--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:45:46.995309Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-313c7-6b795f969b-wmjtk namespace: kserve-ci-e2e-test resourceVersion: "20849" uid: 4af22d05-2d05-48f9-baef-808a639508da kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-313c7-6b795f969b-wmjtk to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:45:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:45:46Z" name: ensemble-graph-313c7-6b795f969b-wmjtk.18a735feeaa0b5dd namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: ee663aff-d1ef-4b33-bd3e-10e4ee118c11 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:47Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-313c7-6b795f969b-wmjtk namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: 4af22d05-2d05-48f9-baef-808a639508da kind: Event lastTimestamp: "2026-04-17T17:45:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-313c7-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:45:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:47Z" name: ensemble-graph-313c7-6b795f969b-wmjtk.18a735fefb33b60b namespace: kserve-ci-e2e-test resourceVersion: "20867" uid: 1199c9a4-f998-494e-a293-3fad084b27b4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:48Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-313c7-6b795f969b-wmjtk namespace: kserve-ci-e2e-test resourceVersion: "20857" uid: 4af22d05-2d05-48f9-baef-808a639508da kind: Event lastTimestamp: "2026-04-17T17:45:48Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:45:48Z" name: ensemble-graph-313c7-6b795f969b-wmjtk.18a735ff27646f83 namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: ea885d0a-24aa-43e2-b0f2-9059009f8289 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-313c7} kind: Pod name: ensemble-graph-313c7-6b795f969b-wmjtk namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: 4af22d05-2d05-48f9-baef-808a639508da kind: Event lastTimestamp: "2026-04-17T17:45:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0589637b57a3315022b8ebf7fc1b0f9831e3a45e07a3fc2fab24137dd8f190f9" already present on machine metadata: creationTimestamp: "2026-04-17T17:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:48Z" name: ensemble-graph-313c7-6b795f969b-wmjtk.18a735ff28d49718 namespace: kserve-ci-e2e-test resourceVersion: "20877" uid: 84659f91-a9fb-4923-9cc1-b30403bce0eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-313c7} kind: Pod name: ensemble-graph-313c7-6b795f969b-wmjtk namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: 4af22d05-2d05-48f9-baef-808a639508da kind: Event lastTimestamp: "2026-04-17T17:45:48Z" message: 'Created container: ensemble-graph-313c7' metadata: creationTimestamp: "2026-04-17T17:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:48Z" name: ensemble-graph-313c7-6b795f969b-wmjtk.18a735ff2c32e95f namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: ae28da9f-c350-4bc1-b997-f6fc4c3d377d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-313c7} kind: Pod name: ensemble-graph-313c7-6b795f969b-wmjtk namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: 4af22d05-2d05-48f9-baef-808a639508da kind: Event lastTimestamp: "2026-04-17T17:45:48Z" message: Started container ensemble-graph-313c7 metadata: creationTimestamp: "2026-04-17T17:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:48Z" name: ensemble-graph-313c7-6b795f969b-wmjtk.18a735ff2ca2fe9e namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: eb2f93d6-2c35-41fa-bd13-730e126a70a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-313c7} kind: Pod name: ensemble-graph-313c7-6b795f969b-wmjtk namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: 4af22d05-2d05-48f9-baef-808a639508da kind: Event lastTimestamp: "2026-04-17T17:54:01Z" message: Stopping container ensemble-graph-313c7 metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:01Z" name: ensemble-graph-313c7-6b795f969b-wmjtk.18a7367216daa523 namespace: kserve-ci-e2e-test resourceVersion: "24365" uid: 61b35278-29da-42c0-a2d8-43d262bb3b03 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-313c7} kind: Pod name: ensemble-graph-313c7-6b795f969b-wmjtk namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: 4af22d05-2d05-48f9-baef-808a639508da kind: Event lastTimestamp: "2026-04-17T17:54:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:29Z" name: ensemble-graph-313c7-6b795f969b-wmjtk.18a73672c50f2861 namespace: kserve-ci-e2e-test resourceVersion: "24698" uid: bcffc153-7d48-4500-a37f-d8ecf4ff662a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-313c7-6b795f969b namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 109db737-4502-4b29-8c32-608b606be557 kind: Event lastTimestamp: "2026-04-17T17:45:46Z" message: 'Created pod: ensemble-graph-313c7-6b795f969b-wmjtk' metadata: creationTimestamp: "2026-04-17T17:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:46Z" name: ensemble-graph-313c7-6b795f969b.18a735feea39e42e namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: fcda4519-fc5e-4300-82f8-ce0c5732c289 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-313c7 namespace: kserve-ci-e2e-test resourceVersion: "20843" uid: 60b34ece-89cf-438c-81c9-2ec934c2b71a kind: Event lastTimestamp: "2026-04-17T17:45:46Z" message: Scaled up replica set ensemble-graph-313c7-6b795f969b from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:45:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:46Z" name: ensemble-graph-313c7.18a735fee9a012ad namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: 2ae26d1d-12f6-4742-9c1d-0d49b6f7ad3d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-313c7 namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 9d572570-f424-4e2c-8c40-d0aab3594145 kind: Event lastTimestamp: "2026-04-17T17:45:47Z" message: InferenceGraph [ensemble-graph-313c7] is Ready metadata: creationTimestamp: "2026-04-17T17:45:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:47Z" name: ensemble-graph-313c7.18a735fef02859b1 namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 20bcd6e8-1a3a-4087-8e11-b9713bf74ed6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:46:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-313c7 namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: b38fcb71-7149-4d34-aede-a1895d3a7233 kind: Event lastTimestamp: "2026-04-17T17:46:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:17Z" name: ensemble-graph-313c7.18a73602680973fe namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: 5fd0f4c0-9112-4c02-bbe8-654ed8d31fd9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:46:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-313c7 namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: b38fcb71-7149-4d34-aede-a1895d3a7233 kind: Event lastTimestamp: "2026-04-17T17:46:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:17Z" name: ensemble-graph-313c7.18a73602686d2b3a namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: 5f79fe81-926c-4f6a-8bf5-e83d19bd0cb0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:44:36.749209Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d1a8d-548898d46-886wj namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 443d72a2-0912-4fb1-9fbf-1146f11b002f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-d1a8d-548898d46-886wj to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:44:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:44:36Z" name: ensemble-graph-d1a8d-548898d46-886wj.18a735ee8fa2483c namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 190ec032-32fd-43c3-8536-80470ccca434 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:37Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-d1a8d-548898d46-886wj namespace: kserve-ci-e2e-test resourceVersion: "19887" uid: 443d72a2-0912-4fb1-9fbf-1146f11b002f kind: Event lastTimestamp: "2026-04-17T17:44:37Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:44:37Z" name: ensemble-graph-d1a8d-548898d46-886wj.18a735eea8fca0df namespace: kserve-ci-e2e-test resourceVersion: "19900" uid: 137323e6-1c19-4407-af9e-482dfd21551c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d1a8d} kind: Pod name: ensemble-graph-d1a8d-548898d46-886wj namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 443d72a2-0912-4fb1-9fbf-1146f11b002f kind: Event lastTimestamp: "2026-04-17T17:44:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0589637b57a3315022b8ebf7fc1b0f9831e3a45e07a3fc2fab24137dd8f190f9" already present on machine metadata: creationTimestamp: "2026-04-17T17:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:37Z" name: ensemble-graph-d1a8d-548898d46-886wj.18a735eeaa4d2030 namespace: kserve-ci-e2e-test resourceVersion: "19902" uid: 042ecdef-037a-41ca-b0a2-143945673ba7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d1a8d} kind: Pod name: ensemble-graph-d1a8d-548898d46-886wj namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 443d72a2-0912-4fb1-9fbf-1146f11b002f kind: Event lastTimestamp: "2026-04-17T17:44:37Z" message: 'Created container: ensemble-graph-d1a8d' metadata: creationTimestamp: "2026-04-17T17:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:37Z" name: ensemble-graph-d1a8d-548898d46-886wj.18a735eead47c629 namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: 8eeb9460-4f36-448d-ac33-5e7ce3a0a2a3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d1a8d} kind: Pod name: ensemble-graph-d1a8d-548898d46-886wj namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 443d72a2-0912-4fb1-9fbf-1146f11b002f kind: Event lastTimestamp: "2026-04-17T17:44:37Z" message: Started container ensemble-graph-d1a8d metadata: creationTimestamp: "2026-04-17T17:44:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:37Z" name: ensemble-graph-d1a8d-548898d46-886wj.18a735eeadb65131 namespace: kserve-ci-e2e-test resourceVersion: "19904" uid: e44c8e1d-8877-43c9-bd76-6f1179eeae92 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d1a8d} kind: Pod name: ensemble-graph-d1a8d-548898d46-886wj namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 443d72a2-0912-4fb1-9fbf-1146f11b002f kind: Event lastTimestamp: "2026-04-17T17:44:46Z" message: Stopping container ensemble-graph-d1a8d metadata: creationTimestamp: "2026-04-17T17:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:46Z" name: ensemble-graph-d1a8d-548898d46-886wj.18a735f0e6ecb704 namespace: kserve-ci-e2e-test resourceVersion: "20010" uid: a1a72a90-9f38-4f93-b92b-1e0f0a47eb98 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:44:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-d1a8d} kind: Pod name: ensemble-graph-d1a8d-548898d46-886wj namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 443d72a2-0912-4fb1-9fbf-1146f11b002f kind: Event lastTimestamp: "2026-04-17T17:45:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:13Z" name: ensemble-graph-d1a8d-548898d46-886wj.18a735f141439061 namespace: kserve-ci-e2e-test resourceVersion: "20351" uid: 2cf651b9-3a26-43c2-9e25-effe6f4c8c4e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-d1a8d-548898d46 namespace: kserve-ci-e2e-test resourceVersion: "19876" uid: d13b53fd-e0e5-4d6e-acf0-caa6d440b1c2 kind: Event lastTimestamp: "2026-04-17T17:44:36Z" message: 'Created pod: ensemble-graph-d1a8d-548898d46-886wj' metadata: creationTimestamp: "2026-04-17T17:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:36Z" name: ensemble-graph-d1a8d-548898d46.18a735ee8f227fbb namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: 11e10b22-3cee-4a14-9208-0c5eecfa029f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-d1a8d namespace: kserve-ci-e2e-test resourceVersion: "19873" uid: 4b3f073b-47af-4841-97e0-c7b6de7051e3 kind: Event lastTimestamp: "2026-04-17T17:44:36Z" message: Scaled up replica set ensemble-graph-d1a8d-548898d46 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:36Z" name: ensemble-graph-d1a8d.18a735ee8e7cb917 namespace: kserve-ci-e2e-test resourceVersion: "19877" uid: 0f71df6e-844b-4054-bcf2-e38071002a18 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-d1a8d namespace: kserve-ci-e2e-test resourceVersion: "19894" uid: bf4673af-0131-4bca-b253-90c748a707b5 kind: Event lastTimestamp: "2026-04-17T17:44:36Z" message: InferenceGraph [ensemble-graph-d1a8d] is Ready metadata: creationTimestamp: "2026-04-17T17:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:44:36Z" name: ensemble-graph-d1a8d.18a735ee95e30abf namespace: kserve-ci-e2e-test resourceVersion: "19896" uid: f4848075-219e-4b11-b2b7-651991bdad42 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:54:51.478396Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42 namespace: kserve-ci-e2e-test resourceVersion: "24978" uid: b5c96035-1ce5-497f-8381-40ed551ff7fe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0283f-predictor-5c787cfb59-k6k42 to ip-10-0-130-19.ec2.internal metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:54:51Z" name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42.18a7367db0595840 namespace: kserve-ci-e2e-test resourceVersion: "24982" uid: 6b870de2-9db5-4eeb-9cad-65519201acda reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42 namespace: kserve-ci-e2e-test resourceVersion: "24988" uid: b5c96035-1ce5-497f-8381-40ed551ff7fe kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:54:51Z" name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42.18a7367dc9c6be2b namespace: kserve-ci-e2e-test resourceVersion: "25001" uid: 0139e588-9127-4b2d-9e65-9125b6e2cf1e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: b5c96035-1ce5-497f-8381-40ed551ff7fe kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:51Z" name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42.18a7367dcb505a95 namespace: kserve-ci-e2e-test resourceVersion: "25003" uid: 0f51b52e-15b2-48c0-810f-c37d5fb3c1db reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: b5c96035-1ce5-497f-8381-40ed551ff7fe kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:51Z" name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42.18a7367dce148029 namespace: kserve-ci-e2e-test resourceVersion: "25004" uid: 0f4aaf6c-ead7-4e64-a5db-92b4f62c7428 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: b5c96035-1ce5-497f-8381-40ed551ff7fe kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:51Z" name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42.18a7367dce8ad5f4 namespace: kserve-ci-e2e-test resourceVersion: "25005" uid: 67719f40-0c13-4a1e-a9ba-55f26ec246b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: b5c96035-1ce5-497f-8381-40ed551ff7fe kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:51Z" name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42.18a7367dce9823ca namespace: kserve-ci-e2e-test resourceVersion: "25006" uid: 52f97d60-665c-438a-9acb-9c28fe974b78 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: b5c96035-1ce5-497f-8381-40ed551ff7fe kind: Event lastTimestamp: "2026-04-17T17:54:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:52Z" name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42.18a7367dd18d8e00 namespace: kserve-ci-e2e-test resourceVersion: "25007" uid: 0fa52fc8-e35d-4c59-9c33-e3ea0efea351 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: b5c96035-1ce5-497f-8381-40ed551ff7fe kind: Event lastTimestamp: "2026-04-17T17:54:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:52Z" name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42.18a7367dd20701e4 namespace: kserve-ci-e2e-test resourceVersion: "25008" uid: 088d595a-aaa2-487a-8966-327c9819c5f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:54:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: b5c96035-1ce5-497f-8381-40ed551ff7fe kind: Event lastTimestamp: "2026-04-17T17:55:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:29Z" name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42.18a7367e3ee60de3 namespace: kserve-ci-e2e-test resourceVersion: "25524" uid: 67ba9932-7efc-47ad-a476-fe78805868d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T18:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: b5c96035-1ce5-497f-8381-40ed551ff7fe kind: Event lastTimestamp: "2026-04-17T18:12:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T18:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:12:10Z" name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42.18a7376fb729b287 namespace: kserve-ci-e2e-test resourceVersion: "32940" uid: 1d09ee6d-0288-4057-bb94-a179a77fafc0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T18:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42 namespace: kserve-ci-e2e-test resourceVersion: "24979" uid: b5c96035-1ce5-497f-8381-40ed551ff7fe kind: Event lastTimestamp: "2026-04-17T18:12:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T18:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:12:10Z" name: error-404-isvc-0283f-predictor-5c787cfb59-k6k42.18a7376fb72a2e81 namespace: kserve-ci-e2e-test resourceVersion: "32941" uid: 66f90655-04a1-4ee1-bdbc-546f4f986ef6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0283f-predictor-5c787cfb59 namespace: kserve-ci-e2e-test resourceVersion: "24971" uid: 06afbc7b-c3dd-44d4-9524-18427063215b kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: 'Created pod: error-404-isvc-0283f-predictor-5c787cfb59-k6k42' metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:54:51Z" name: error-404-isvc-0283f-predictor-5c787cfb59.18a7367daf97ddfe namespace: kserve-ci-e2e-test resourceVersion: "24980" uid: c5e94e13-fb15-4431-9bc5-bdc2262e5e75 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0283f-predictor namespace: kserve-ci-e2e-test resourceVersion: "24969" uid: f424d20c-7c93-4697-9144-53237fc5ea25 kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: Scaled up replica set error-404-isvc-0283f-predictor-5c787cfb59 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:54:51Z" name: error-404-isvc-0283f-predictor.18a7367dad738919 namespace: kserve-ci-e2e-test resourceVersion: "24973" uid: 2fb856d0-773f-45fc-9609-e4f0c0b1c7b8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:55:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0283f-predictor namespace: kserve-ci-e2e-test resourceVersion: "24970" uid: 0767cfd2-2d9d-452e-afe8-f97ffa203bc5 kind: Event lastTimestamp: "2026-04-17T17:55:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:21Z" name: error-404-isvc-0283f-predictor.18a736812bf37dd8 namespace: kserve-ci-e2e-test resourceVersion: "25456" uid: edfef806-bbc2-421a-8b5e-02846b2e7c43 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:55:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0283f-predictor namespace: kserve-ci-e2e-test resourceVersion: "24970" uid: 0767cfd2-2d9d-452e-afe8-f97ffa203bc5 kind: Event lastTimestamp: "2026-04-17T17:55:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:21Z" name: error-404-isvc-0283f-predictor.18a736812c67203b namespace: kserve-ci-e2e-test resourceVersion: "25457" uid: 506fef96-b46d-46a0-9286-0778c0bd69fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:55:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0283f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25185" uid: 0767cfd2-2d9d-452e-afe8-f97ffa203bc5 kind: Event lastTimestamp: "2026-04-17T17:55:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:51Z" name: error-404-isvc-0283f-predictor.18a7368829e1f3b3 namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 46da32d3-d351-4603-9ff3-cf9b2bc037d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:55:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0283f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25185" uid: 0767cfd2-2d9d-452e-afe8-f97ffa203bc5 kind: Event lastTimestamp: "2026-04-17T17:55:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:51Z" name: error-404-isvc-0283f-predictor.18a736882a4dcd19 namespace: kserve-ci-e2e-test resourceVersion: "25739" uid: 515332b5-b721-41e7-83a1-4f7e0dde236c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0283f namespace: kserve-ci-e2e-test resourceVersion: "24944" uid: 95781d78-1117-438b-8d93-4d187ff4b7c8 kind: Event lastTimestamp: "2026-04-17T17:55:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:55:39Z" name: error-404-isvc-0283f.18a7367dad4f7f0d namespace: kserve-ci-e2e-test resourceVersion: "25616" uid: 5cf470b3-942e-4db6-848f-abaaf09c1e6c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0283f namespace: kserve-ci-e2e-test resourceVersion: "24944" uid: 95781d78-1117-438b-8d93-4d187ff4b7c8 kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-0283f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0283f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:54:51Z" name: error-404-isvc-0283f.18a7367dada12f8c namespace: kserve-ci-e2e-test resourceVersion: "24975" uid: 9cc6f8ca-e6d1-4883-95f3-42abc45245bf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0283f namespace: kserve-ci-e2e-test resourceVersion: "24944" uid: 95781d78-1117-438b-8d93-4d187ff4b7c8 kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0283f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:54:51Z" name: error-404-isvc-0283f.18a7367dada17258 namespace: kserve-ci-e2e-test resourceVersion: "24976" uid: 58afd2d9-a559-446c-ba32-3cf7a4c44749 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0283f namespace: kserve-ci-e2e-test resourceVersion: "25614" uid: 95781d78-1117-438b-8d93-4d187ff4b7c8 kind: Event lastTimestamp: "2026-04-17T17:55:39Z" message: InferenceService [error-404-isvc-0283f] is Ready metadata: creationTimestamp: "2026-04-17T17:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:55:39Z" name: error-404-isvc-0283f.18a73688fa1bf62a namespace: kserve-ci-e2e-test resourceVersion: "25615" uid: 0e8431c3-7704-4024-8004-36482774fbe6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:32:51.749703Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13772" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb to ip-10-0-130-19.ec2.internal metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:32:51Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7354a6a6397a2 namespace: kserve-ci-e2e-test resourceVersion: "13776" uid: d0d66be4-f4c5-469f-8633-68e5c1fe2f1f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-235f3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:32:51Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7354a7484fa9f namespace: kserve-ci-e2e-test resourceVersion: "13787" uid: ca3b4489-915a-47b3-8dc3-64e02d2d7f2b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13781" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:32:52Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:32:52Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7354aa76cff93 namespace: kserve-ci-e2e-test resourceVersion: "13828" uid: a2160ba3-14d4-4ff1-b573-7a37eddd6bd9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:32:52Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" metadata: creationTimestamp: "2026-04-17T17:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:32:52Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7354aa8c25bbc namespace: kserve-ci-e2e-test resourceVersion: "13830" uid: 61e9e982-5438-4256-8d93-2cdcc0c0dc37 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:33:05Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" in 12.294s (12.294s including waiting). Image size: 1332164705 bytes.' metadata: creationTimestamp: "2026-04-17T17:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:05Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7354d858b5adb namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: ca440f21-1025-4284-8671-146704f83418 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:33:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:05Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7354d891d2011 namespace: kserve-ci-e2e-test resourceVersion: "13979" uid: 3ad23045-140b-42e4-9de8-82d7aa400b04 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:33:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:05Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7354d89ced175 namespace: kserve-ci-e2e-test resourceVersion: "13980" uid: 61c05e26-c8bd-4f69-b754-73e7ff283074 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:33:05Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-17T17:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:05Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7354d89df6fa9 namespace: kserve-ci-e2e-test resourceVersion: "13981" uid: 1da1564c-d05d-4a34-87a0-28dca24ae049 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:33:12Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 7.273s (7.273s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-17T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:12Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7354f3b667c0b namespace: kserve-ci-e2e-test resourceVersion: "14058" uid: b2622ce2-6012-4d9b-952c-f2133118a7f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:33:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:12Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7354f3f5e1682 namespace: kserve-ci-e2e-test resourceVersion: "14062" uid: db858271-8ef9-46f5-b99c-293b2ad4a8c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:33:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:12Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7354f4000e76f namespace: kserve-ci-e2e-test resourceVersion: "14064" uid: d986012f-7861-4e54-b8f2-06b709f73cd6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:33:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:33:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:33:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:49Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7354f75ca344d namespace: kserve-ci-e2e-test resourceVersion: "14393" uid: 722abfb7-0a2f-432b-869e-0c61ecefd8e4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:21Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7355f56e5aa11 namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: dcc977e5-84c8-4bda-9813-1f45c6ce03a9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:21Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7355f56e628aa namespace: kserve-ci-e2e-test resourceVersion: "14788" uid: 490f8bd5-0f86-41f3-a56a-3c6e2ad51e8a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: 0fbffdfe-842e-4cb3-850f-d16083151446 kind: Event lastTimestamp: "2026-04-17T17:34:24Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:34:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:24Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb.18a7355ffddf7477 namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: 157e2929-72ef-41ca-adf5-a8f1493dd961 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-235f3-predictor-6d9f8fbcff namespace: kserve-ci-e2e-test resourceVersion: "13761" uid: 7a2f8eab-d7ae-47dc-9164-129469a53064 kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: 'Created pod: error-404-isvc-235f3-predictor-6d9f8fbcff-zftlb' metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:32:51Z" name: error-404-isvc-235f3-predictor-6d9f8fbcff.18a7354a69eb2d61 namespace: kserve-ci-e2e-test resourceVersion: "13773" uid: d007e1a4-4798-42c0-bdcf-49448888a58e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-235f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "13759" uid: 1ff9cf37-6612-469e-a4e0-722a805ad662 kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: Scaled up replica set error-404-isvc-235f3-predictor-6d9f8fbcff from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:32:51Z" name: error-404-isvc-235f3-predictor.18a7354a641d23d8 namespace: kserve-ci-e2e-test resourceVersion: "13765" uid: ef730b73-a6ab-4fb3-9520-a12d1ec48ee5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-235f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "13760" uid: 30facad6-295a-4bee-aa55-7e2b6a7b5b12 kind: Event lastTimestamp: "2026-04-17T17:33:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:33:36Z" name: error-404-isvc-235f3-predictor.18a7354de2a7bc8c namespace: kserve-ci-e2e-test resourceVersion: "14304" uid: d9167a57-9f0f-4655-9503-7da5d7842e3c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-235f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "13760" uid: 30facad6-295a-4bee-aa55-7e2b6a7b5b12 kind: Event lastTimestamp: "2026-04-17T17:33:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:33:36Z" name: error-404-isvc-235f3-predictor.18a7354de3122bf3 namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: e46d34b2-04f6-4426-bbab-5d7a6f84edf5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-235f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 30facad6-295a-4bee-aa55-7e2b6a7b5b12 kind: Event lastTimestamp: "2026-04-17T17:34:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:06Z" name: error-404-isvc-235f3-predictor.18a735586065ef77 namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: faf53337-a6d9-4f76-a423-c4c957810dab reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-235f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: 30facad6-295a-4bee-aa55-7e2b6a7b5b12 kind: Event lastTimestamp: "2026-04-17T17:34:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:06Z" name: error-404-isvc-235f3-predictor.18a7355860d20f58 namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 7d25af56-d1f4-498a-acab-1c47685c4f09 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-235f3 namespace: kserve-ci-e2e-test resourceVersion: "13743" uid: 8bd43f95-c3fc-49ab-96c0-40951f259685 kind: Event lastTimestamp: "2026-04-17T17:33:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:33:59Z" name: error-404-isvc-235f3.18a7354a6404d053 namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: 80b39490-0f37-4a79-a9ba-baddc4543fbb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-235f3 namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 8bd43f95-c3fc-49ab-96c0-40951f259685 kind: Event lastTimestamp: "2026-04-17T17:33:59Z" message: InferenceService [error-404-isvc-235f3] is Ready metadata: creationTimestamp: "2026-04-17T17:33:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:33:59Z" name: error-404-isvc-235f3.18a7355a30f86c32 namespace: kserve-ci-e2e-test resourceVersion: "14499" uid: 93bbbd5a-e43d-4a81-a3d5-8419150c624d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:44:47.121677Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx namespace: kserve-ci-e2e-test resourceVersion: "20090" uid: e4155761-7757-439a-a6ab-446a33381869 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-313c7-predictor-5555fd9c67-rqprx to ip-10-0-130-19.ec2.internal metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx.18a735f0f9e19a6d namespace: kserve-ci-e2e-test resourceVersion: "20094" uid: 3ff18e3c-2126-4ec2-a182-5a9a1df40d62 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: e4155761-7757-439a-a6ab-446a33381869 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx.18a735f112ee2126 namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: 19ff4958-788b-4fe8-8ce6-75fa74b99654 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: e4155761-7757-439a-a6ab-446a33381869 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx.18a735f114427e69 namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: e71e971e-7388-419b-943d-6e4295357543 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: e4155761-7757-439a-a6ab-446a33381869 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx.18a735f116f716e2 namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: 083e44e7-8ee3-4ef0-b7e5-e207884b120f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: e4155761-7757-439a-a6ab-446a33381869 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx.18a735f1176f32e0 namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: bd91c5dd-f6bc-4b73-a931-442da0e5c2e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: e4155761-7757-439a-a6ab-446a33381869 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx.18a735f1177d4bd8 namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 59b78c51-16a9-48cc-8070-550ec7153375 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: e4155761-7757-439a-a6ab-446a33381869 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx.18a735f11a907ed6 namespace: kserve-ci-e2e-test resourceVersion: "20121" uid: 7ca78498-debf-4b89-a690-e044e82204c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: e4155761-7757-439a-a6ab-446a33381869 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx.18a735f11b017df3 namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 515ce887-e3b7-4221-8e93-e5c371817365 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T17:44:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: e4155761-7757-439a-a6ab-446a33381869 kind: Event lastTimestamp: "2026-04-17T17:54:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:04Z" name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx.18a735f15e86280d namespace: kserve-ci-e2e-test resourceVersion: "24509" uid: 2bd78009-1c2d-45b4-ae76-5d7796fae17b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: e4155761-7757-439a-a6ab-446a33381869 kind: Event lastTimestamp: "2026-04-17T17:54:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:01Z" name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx.18a736722137f829 namespace: kserve-ci-e2e-test resourceVersion: "24416" uid: dc4e7e46-baaf-4dec-81e9-615609c73180 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: e4155761-7757-439a-a6ab-446a33381869 kind: Event lastTimestamp: "2026-04-17T17:54:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:01Z" name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx.18a736722138bd0d namespace: kserve-ci-e2e-test resourceVersion: "24419" uid: c9a4dc83-5dc9-48b4-8bff-4a8be0a6b25d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: e4155761-7757-439a-a6ab-446a33381869 kind: Event lastTimestamp: "2026-04-17T17:54:04Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:54:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:04Z" name: error-404-isvc-313c7-predictor-5555fd9c67-rqprx.18a73672d2e51277 namespace: kserve-ci-e2e-test resourceVersion: "24508" uid: a0dd099f-faa0-4fc5-86f4-a40bde2c515a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-313c7-predictor-5555fd9c67 namespace: kserve-ci-e2e-test resourceVersion: "20081" uid: bf4a5fd8-7c5f-44c2-811b-76ba2b9c7543 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: 'Created pod: error-404-isvc-313c7-predictor-5555fd9c67-rqprx' metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-313c7-predictor-5555fd9c67.18a735f0f8bbd4d1 namespace: kserve-ci-e2e-test resourceVersion: "20091" uid: 932f6eab-1f81-49c8-88cf-4db00aa1d95a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-313c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20078" uid: 20b31aa1-92b6-4f40-b188-bd5c54ce2394 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: Scaled up replica set error-404-isvc-313c7-predictor-5555fd9c67 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-313c7-predictor.18a735f0f4ef8cc4 namespace: kserve-ci-e2e-test resourceVersion: "20082" uid: 2e498ff7-109c-4aae-8a1e-dbc3354d859e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:45:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-313c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20080" uid: 399ad987-f25d-4b62-9940-9391bd9d262d kind: Event lastTimestamp: "2026-04-17T17:45:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:17Z" name: error-404-isvc-313c7-predictor.18a735f4738ddfc9 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: abecd5d6-bdb9-41ed-b071-afce088ce6b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:45:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-313c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20080" uid: 399ad987-f25d-4b62-9940-9391bd9d262d kind: Event lastTimestamp: "2026-04-17T17:45:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:17Z" name: error-404-isvc-313c7-predictor.18a735f473f7dbbf namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: b08511b3-2aed-4e38-9b4d-87faa1dd7a85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:45:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-313c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20263" uid: 399ad987-f25d-4b62-9940-9391bd9d262d kind: Event lastTimestamp: "2026-04-17T17:46:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:02Z" name: error-404-isvc-313c7-predictor.18a735fb71be8432 namespace: kserve-ci-e2e-test resourceVersion: "20999" uid: 866dcb47-1d57-4491-9336-703b7b20f7a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:45:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-313c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20263" uid: 399ad987-f25d-4b62-9940-9391bd9d262d kind: Event lastTimestamp: "2026-04-17T17:46:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:45:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:02Z" name: error-404-isvc-313c7-predictor.18a735fb723086c2 namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: 14658d09-1b51-463b-b085-9d6826efb6e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-313c7 namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: 2e8bc7ea-1cb5-49b2-90e1-78ba2b442a3a kind: Event lastTimestamp: "2026-04-17T17:45:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:34Z" name: error-404-isvc-313c7.18a735f0f4d7dde9 namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: 17b6ae93-b110-48a0-bd83-f9eadf07fe96 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-313c7 namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: 2e8bc7ea-1cb5-49b2-90e1-78ba2b442a3a kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: 'Failed to update status for InferenceService "error-404-isvc-313c7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-313c7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-313c7.18a735f0f672b756 namespace: kserve-ci-e2e-test resourceVersion: "20087" uid: 4353732d-64bd-4615-8288-deaac4028482 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-313c7 namespace: kserve-ci-e2e-test resourceVersion: "20052" uid: 2e8bc7ea-1cb5-49b2-90e1-78ba2b442a3a kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-313c7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-313c7.18a735f0f672f15b namespace: kserve-ci-e2e-test resourceVersion: "20089" uid: 565d9ee2-6ceb-415c-9226-fc77be27109e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-313c7 namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: 2e8bc7ea-1cb5-49b2-90e1-78ba2b442a3a kind: Event lastTimestamp: "2026-04-17T17:45:34Z" message: InferenceService [error-404-isvc-313c7] is Ready metadata: creationTimestamp: "2026-04-17T17:45:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:34Z" name: error-404-isvc-313c7.18a735fc1a67745c namespace: kserve-ci-e2e-test resourceVersion: "20586" uid: 3b671dc3-282a-41bf-9ba0-58f15a75c4ce reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:44:26.458244Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp namespace: kserve-ci-e2e-test resourceVersion: "19742" uid: b8534ca9-5882-4a61-8cc9-35cddd3eb42b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a1277-predictor-8558bb5997-nxrxp to ip-10-0-130-19.ec2.internal metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:44:26Z" name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp.18a735ec2a3e9bb3 namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: affdc844-c58a-4c17-a95c-e72f76a3609f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: b8534ca9-5882-4a61-8cc9-35cddd3eb42b kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a1277-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:26Z" name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp.18a735ec36550614 namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: 9013d264-8de2-4e72-817f-621a5a9ee223 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: b8534ca9-5882-4a61-8cc9-35cddd3eb42b kind: Event lastTimestamp: "2026-04-17T17:44:27Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:44:27Z" name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp.18a735ec67fca502 namespace: kserve-ci-e2e-test resourceVersion: "19778" uid: 4cf08566-5392-4190-be2a-beba8dda3ba4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: b8534ca9-5882-4a61-8cc9-35cddd3eb42b kind: Event lastTimestamp: "2026-04-17T17:44:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:27Z" name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp.18a735ec6995a178 namespace: kserve-ci-e2e-test resourceVersion: "19780" uid: 48ad0e3e-03c4-453d-94f9-420f1490a508 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: b8534ca9-5882-4a61-8cc9-35cddd3eb42b kind: Event lastTimestamp: "2026-04-17T17:44:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:27Z" name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp.18a735ec6c95ea44 namespace: kserve-ci-e2e-test resourceVersion: "19782" uid: b2a05100-a5b3-4785-8653-a7d7a2630ec9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: b8534ca9-5882-4a61-8cc9-35cddd3eb42b kind: Event lastTimestamp: "2026-04-17T17:44:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:27Z" name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp.18a735ec6d096b9f namespace: kserve-ci-e2e-test resourceVersion: "19783" uid: 2f0f8b4b-6275-41b8-a0a9-1e63036ebf43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: b8534ca9-5882-4a61-8cc9-35cddd3eb42b kind: Event lastTimestamp: "2026-04-17T17:44:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:27Z" name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp.18a735ec6d165745 namespace: kserve-ci-e2e-test resourceVersion: "19784" uid: ae468ec1-a8d5-4562-97b4-30a8748eb12b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: b8534ca9-5882-4a61-8cc9-35cddd3eb42b kind: Event lastTimestamp: "2026-04-17T17:44:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:27Z" name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp.18a735ec703639f5 namespace: kserve-ci-e2e-test resourceVersion: "19785" uid: f65e0b76-19c5-416c-b065-0815cd3cc1ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: b8534ca9-5882-4a61-8cc9-35cddd3eb42b kind: Event lastTimestamp: "2026-04-17T17:44:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:27Z" name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp.18a735ec70ba3e5c namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: d66c0772-6685-42ef-ba43-18e0429e67e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:44:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: b8534ca9-5882-4a61-8cc9-35cddd3eb42b kind: Event lastTimestamp: "2026-04-17T17:45:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:04Z" name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp.18a735ecb198c920 namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: eb07a2b7-ced0-4643-9fc5-b3fef95779a4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: b8534ca9-5882-4a61-8cc9-35cddd3eb42b kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:36Z" name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp.18a735fc7df61815 namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 470b9e11-e84c-4fe8-a18a-7799620857bb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: b8534ca9-5882-4a61-8cc9-35cddd3eb42b kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:36Z" name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp.18a735fc7df72e70 namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: 3f778c42-1e9c-4227-8f2c-5357125f75f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: b8534ca9-5882-4a61-8cc9-35cddd3eb42b kind: Event lastTimestamp: "2026-04-17T17:45:39Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:39Z" name: error-404-isvc-a1277-predictor-8558bb5997-nxrxp.18a735fd39c5bf3a namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: 7ddb9ec7-7894-4789-be50-105022e1146d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a1277-predictor-8558bb5997 namespace: kserve-ci-e2e-test resourceVersion: "19735" uid: fe1d478c-18dd-4037-91e4-44908df933cd kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: 'Created pod: error-404-isvc-a1277-predictor-8558bb5997-nxrxp' metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:26Z" name: error-404-isvc-a1277-predictor-8558bb5997.18a735ec29a64498 namespace: kserve-ci-e2e-test resourceVersion: "19743" uid: af5c53bd-072b-4cb7-9100-a67331e8a00b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a1277-predictor namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: dd4e31ff-04f7-41e1-a623-17b42ec37061 kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: Scaled up replica set error-404-isvc-a1277-predictor-8558bb5997 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:26Z" name: error-404-isvc-a1277-predictor.18a735ec27d8ec9b namespace: kserve-ci-e2e-test resourceVersion: "19737" uid: ac85f304-226c-4980-ab0a-68556e1ecfa1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a1277-predictor namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: b3f27751-1220-45e2-9e26-7fcf92ffe6d4 kind: Event lastTimestamp: "2026-04-17T17:44:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:41Z" name: error-404-isvc-a1277-predictor.18a735efa6739f30 namespace: kserve-ci-e2e-test resourceVersion: "19945" uid: bae8a838-c97c-47b5-b0ec-98159919e782 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a1277-predictor namespace: kserve-ci-e2e-test resourceVersion: "19734" uid: b3f27751-1220-45e2-9e26-7fcf92ffe6d4 kind: Event lastTimestamp: "2026-04-17T17:44:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:41Z" name: error-404-isvc-a1277-predictor.18a735efa6e316f2 namespace: kserve-ci-e2e-test resourceVersion: "19947" uid: 734453fa-26d8-4af6-8c4d-ac9fc7bf6562 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:44:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a1277-predictor namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: b3f27751-1220-45e2-9e26-7fcf92ffe6d4 kind: Event lastTimestamp: "2026-04-17T17:45:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:26Z" name: error-404-isvc-a1277-predictor.18a735f32590bbee namespace: kserve-ci-e2e-test resourceVersion: "20501" uid: 2ec08f61-c8db-48d2-b9cf-6aae7f1569fe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:44:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a1277-predictor namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: b3f27751-1220-45e2-9e26-7fcf92ffe6d4 kind: Event lastTimestamp: "2026-04-17T17:45:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:44:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:26Z" name: error-404-isvc-a1277-predictor.18a735f3260b04d1 namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 560981e3-8c57-46d3-b60d-06dfa9125619 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a1277 namespace: kserve-ci-e2e-test resourceVersion: "19701" uid: dac2b33e-1eca-4452-8dba-d781c03aadd7 kind: Event lastTimestamp: "2026-04-17T17:45:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:14Z" name: error-404-isvc-a1277.18a735ec27b688b8 namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: 4bd3dcbb-22b3-480d-a098-59d39d1bc08a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a1277 namespace: kserve-ci-e2e-test resourceVersion: "19701" uid: dac2b33e-1eca-4452-8dba-d781c03aadd7 kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-a1277": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a1277": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:44:26Z" name: error-404-isvc-a1277.18a735ec2824c3c4 namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: d6f17464-93c4-4b29-a73b-1718f69bd105 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a1277 namespace: kserve-ci-e2e-test resourceVersion: "19701" uid: dac2b33e-1eca-4452-8dba-d781c03aadd7 kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a1277": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:44:26Z" name: error-404-isvc-a1277.18a735ec28251f92 namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: e56c27d1-c7c6-45d2-8a6f-aa374651b8b4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a1277 namespace: kserve-ci-e2e-test resourceVersion: "20382" uid: dac2b33e-1eca-4452-8dba-d781c03aadd7 kind: Event lastTimestamp: "2026-04-17T17:45:14Z" message: InferenceService [error-404-isvc-a1277] is Ready metadata: creationTimestamp: "2026-04-17T17:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:14Z" name: error-404-isvc-a1277.18a735f76d0813a3 namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 742b2925-d466-4f5f-b2b0-0bc836ca43c2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:55:12.199142Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt namespace: kserve-ci-e2e-test resourceVersion: "25329" uid: a249c6aa-09d4-40df-b18b-674b50f2e08e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt to ip-10-0-130-19.ec2.internal metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt.18a736828366d33f namespace: kserve-ci-e2e-test resourceVersion: "25333" uid: 501d6491-5ac7-4093-98cf-dc4260c57d7a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt namespace: kserve-ci-e2e-test resourceVersion: "25340" uid: a249c6aa-09d4-40df-b18b-674b50f2e08e kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt.18a736829cc6c531 namespace: kserve-ci-e2e-test resourceVersion: "25359" uid: 2ab1b432-8daa-4104-a482-14a2b1f22939 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: a249c6aa-09d4-40df-b18b-674b50f2e08e kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt.18a736829e151391 namespace: kserve-ci-e2e-test resourceVersion: "25361" uid: 7ee7ad41-70f0-4650-b393-99799a83a444 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: a249c6aa-09d4-40df-b18b-674b50f2e08e kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt.18a73682a0e7279a namespace: kserve-ci-e2e-test resourceVersion: "25364" uid: 343134b2-4c13-4f92-b50b-d99febfe09cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: a249c6aa-09d4-40df-b18b-674b50f2e08e kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt.18a73682a1d8e4e7 namespace: kserve-ci-e2e-test resourceVersion: "25365" uid: 49718538-e803-48ea-84bf-dcf7cb5a8abe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: a249c6aa-09d4-40df-b18b-674b50f2e08e kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt.18a73682a1e7a22a namespace: kserve-ci-e2e-test resourceVersion: "25366" uid: f4d24cb5-c5e6-4d4a-aeee-1cca92d24d9b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: a249c6aa-09d4-40df-b18b-674b50f2e08e kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt.18a73682a53615f8 namespace: kserve-ci-e2e-test resourceVersion: "25369" uid: 38516daf-161d-4bc4-a1d6-08134796cdb8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: a249c6aa-09d4-40df-b18b-674b50f2e08e kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt.18a73682a5aec55c namespace: kserve-ci-e2e-test resourceVersion: "25370" uid: 335c9680-7ed5-4ccd-b113-0458a42ed5b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: a249c6aa-09d4-40df-b18b-674b50f2e08e kind: Event lastTimestamp: "2026-04-17T18:04:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:04:28Z" name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt.18a73682afc343f1 namespace: kserve-ci-e2e-test resourceVersion: "29740" uid: 7bb45ddf-9bc2-4412-a1e4-dcac81b231b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T18:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: a249c6aa-09d4-40df-b18b-674b50f2e08e kind: Event lastTimestamp: "2026-04-17T18:04:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T18:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:04:26Z" name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt.18a73703a6bd00a4 namespace: kserve-ci-e2e-test resourceVersion: "29724" uid: 91051cc3-188f-452f-bfd1-81bf631d46da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T18:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: a249c6aa-09d4-40df-b18b-674b50f2e08e kind: Event lastTimestamp: "2026-04-17T18:04:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T18:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:04:26Z" name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt.18a73703a6bd588c namespace: kserve-ci-e2e-test resourceVersion: "29725" uid: 15fd71a8-ad6c-435a-ac88-a3a1f453b155 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T18:04:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt namespace: kserve-ci-e2e-test resourceVersion: "25331" uid: a249c6aa-09d4-40df-b18b-674b50f2e08e kind: Event lastTimestamp: "2026-04-17T18:04:28Z" message: 'Readiness probe failed: Get "https://10.133.0.42:8643/healthz": dial tcp 10.133.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T18:04:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:04:28Z" name: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt.18a73704241fc7ee namespace: kserve-ci-e2e-test resourceVersion: "29739" uid: fbabf06a-add4-4cc0-8a8b-64c90595d0dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a935a-predictor-79fc75c4ff namespace: kserve-ci-e2e-test resourceVersion: "25320" uid: b6dc1023-9bc7-45d8-a8b7-95998ca0276b kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: 'Created pod: error-404-isvc-a935a-predictor-79fc75c4ff-vbnnt' metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-a935a-predictor-79fc75c4ff.18a736828307d26f namespace: kserve-ci-e2e-test resourceVersion: "25330" uid: 799c7b48-88d4-4283-9744-154f527c791e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a935a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25317" uid: 60b32095-0061-4e59-9fad-d976b268c562 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Scaled up replica set error-404-isvc-a935a-predictor-79fc75c4ff from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-a935a-predictor.18a736828178e4fa namespace: kserve-ci-e2e-test resourceVersion: "25323" uid: 5785bdac-5992-4a85-a2be-fe881843640b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:55:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a935a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25319" uid: bb173ebb-d103-48d6-9dc8-dbedcbd63e55 kind: Event lastTimestamp: "2026-04-17T17:55:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:42Z" name: error-404-isvc-a935a-predictor.18a73685fff3a4df namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: 9ef88061-f17d-462f-9143-1adaf9af2f06 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:55:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a935a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25319" uid: bb173ebb-d103-48d6-9dc8-dbedcbd63e55 kind: Event lastTimestamp: "2026-04-17T17:55:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:42Z" name: error-404-isvc-a935a-predictor.18a73686005d401e namespace: kserve-ci-e2e-test resourceVersion: "25646" uid: e6916a90-0d7d-401d-8bd5-d9263472b76f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:55:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a935a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: bb173ebb-d103-48d6-9dc8-dbedcbd63e55 kind: Event lastTimestamp: "2026-04-17T17:56:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:27Z" name: error-404-isvc-a935a-predictor.18a7368cfdea7212 namespace: kserve-ci-e2e-test resourceVersion: "26244" uid: fe4c44bb-f1e6-4db2-b86d-2f9eaf865678 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:55:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a935a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25501" uid: bb173ebb-d103-48d6-9dc8-dbedcbd63e55 kind: Event lastTimestamp: "2026-04-17T17:56:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:27Z" name: error-404-isvc-a935a-predictor.18a7368cfe594400 namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: adf691c6-0678-4add-b51f-f297875493fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a935a namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: 8eda0d70-5e2c-4f57-8ee4-fad89ce93a9e kind: Event lastTimestamp: "2026-04-17T17:55:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:55:59Z" name: error-404-isvc-a935a.18a736828154ff8f namespace: kserve-ci-e2e-test resourceVersion: "25822" uid: b845ea93-e068-4001-a40b-84c8b6cae38a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a935a namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: 8eda0d70-5e2c-4f57-8ee4-fad89ce93a9e kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-a935a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a935a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-a935a.18a7368281bebfed namespace: kserve-ci-e2e-test resourceVersion: "25326" uid: 50f70642-e40b-442b-b3a7-223acbb90d9e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a935a namespace: kserve-ci-e2e-test resourceVersion: "25293" uid: 8eda0d70-5e2c-4f57-8ee4-fad89ce93a9e kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a935a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-a935a.18a7368281bf11eb namespace: kserve-ci-e2e-test resourceVersion: "25327" uid: 9c387efe-e00b-4f87-afe2-449bb1233d3f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a935a namespace: kserve-ci-e2e-test resourceVersion: "25820" uid: 8eda0d70-5e2c-4f57-8ee4-fad89ce93a9e kind: Event lastTimestamp: "2026-04-17T17:55:59Z" message: InferenceService [error-404-isvc-a935a] is Ready metadata: creationTimestamp: "2026-04-17T17:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:55:59Z" name: error-404-isvc-a935a.18a7368d6b3c24ba namespace: kserve-ci-e2e-test resourceVersion: "25821" uid: 9ad46a47-e1d8-499f-8ed4-db0b539322cb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:43:36.757083Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b namespace: kserve-ci-e2e-test resourceVersion: "19199" uid: 82747371-50d2-4c12-8f87-ecc7784c2781 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b to ip-10-0-130-19.ec2.internal metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:43:36Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b.18a735e097d326e9 namespace: kserve-ci-e2e-test resourceVersion: "19203" uid: 0fdc5603-5adc-4d44-abe7-622e18a365e2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: 82747371-50d2-4c12-8f87-ecc7784c2781 kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d1a8d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:36Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b.18a735e0a2aab403 namespace: kserve-ci-e2e-test resourceVersion: "19213" uid: 5947465f-20a7-4dfa-9cfb-9f6db15d5bec reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b namespace: kserve-ci-e2e-test resourceVersion: "19209" uid: 82747371-50d2-4c12-8f87-ecc7784c2781 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:43:37Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b.18a735e0d4f8e46c namespace: kserve-ci-e2e-test resourceVersion: "19231" uid: 77f700b3-3e3f-4b36-9cbe-9f22a246fc63 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: 82747371-50d2-4c12-8f87-ecc7784c2781 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:37Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b.18a735e0d688b8a1 namespace: kserve-ci-e2e-test resourceVersion: "19234" uid: 774d0051-2852-4db4-97c1-8dd0464fa06d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: 82747371-50d2-4c12-8f87-ecc7784c2781 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:37Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b.18a735e0d99f6396 namespace: kserve-ci-e2e-test resourceVersion: "19237" uid: 4181c4f7-8a9e-47e1-a41e-86fc291f36c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: 82747371-50d2-4c12-8f87-ecc7784c2781 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:37Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b.18a735e0da10d94a namespace: kserve-ci-e2e-test resourceVersion: "19238" uid: 9b685e12-3a96-491d-ab3c-95e0d3ecc7c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: 82747371-50d2-4c12-8f87-ecc7784c2781 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:37Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b.18a735e0da1d1ff2 namespace: kserve-ci-e2e-test resourceVersion: "19240" uid: 57926ff1-9780-40e5-a41b-637a49cace60 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: 82747371-50d2-4c12-8f87-ecc7784c2781 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:37Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b.18a735e0dd3474f9 namespace: kserve-ci-e2e-test resourceVersion: "19243" uid: d5c11773-116a-4a5e-aa3a-9ee219e7caf6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: 82747371-50d2-4c12-8f87-ecc7784c2781 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:37Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b.18a735e0ddab8d92 namespace: kserve-ci-e2e-test resourceVersion: "19244" uid: fb0fe77c-359b-466a-b7ad-c74b06413f89 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:43:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: 82747371-50d2-4c12-8f87-ecc7784c2781 kind: Event lastTimestamp: "2026-04-17T17:44:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:14Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b.18a735e103215473 namespace: kserve-ci-e2e-test resourceVersion: "19542" uid: 21cc805f-97ca-4add-aeed-dc8a93e60ab8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: 82747371-50d2-4c12-8f87-ecc7784c2781 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b.18a735f0f4b6ee1c namespace: kserve-ci-e2e-test resourceVersion: "20084" uid: 3c47095e-ef8e-468b-a2dd-f578a457d69a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: 82747371-50d2-4c12-8f87-ecc7784c2781 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b.18a735f0f4b7298c namespace: kserve-ci-e2e-test resourceVersion: "20086" uid: 3b5c215b-1e30-49ad-86ea-dc35f8cb9115 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b namespace: kserve-ci-e2e-test resourceVersion: "19201" uid: 82747371-50d2-4c12-8f87-ecc7784c2781 kind: Event lastTimestamp: "2026-04-17T17:44:49Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:49Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b.18a735f18b41c213 namespace: kserve-ci-e2e-test resourceVersion: "20155" uid: 41187195-0f1a-499f-978d-01f03b375c29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d1a8d-predictor-8544d9ff69 namespace: kserve-ci-e2e-test resourceVersion: "19191" uid: 355eb74e-a43c-4c77-9dc1-6e5dbbd548ce kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: 'Created pod: error-404-isvc-d1a8d-predictor-8544d9ff69-9bp5b' metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:43:36Z" name: error-404-isvc-d1a8d-predictor-8544d9ff69.18a735e09762b6ca namespace: kserve-ci-e2e-test resourceVersion: "19200" uid: 52db752c-5479-426c-8a5f-bb05fc147501 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d1a8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19187" uid: 5f0e2467-86c3-4a9e-be10-3b8edaee82e6 kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: Scaled up replica set error-404-isvc-d1a8d-predictor-8544d9ff69 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:43:36Z" name: error-404-isvc-d1a8d-predictor.18a735e095d48d11 namespace: kserve-ci-e2e-test resourceVersion: "19192" uid: 0567f122-d8ff-4e1e-abfe-44f58747a767 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:43:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1a8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19190" uid: cbe264c0-6ad6-4eaf-b3bf-ab0908bae8ed kind: Event lastTimestamp: "2026-04-17T17:44:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:06Z" name: error-404-isvc-d1a8d-predictor.18a735e4148de44c namespace: kserve-ci-e2e-test resourceVersion: "19484" uid: 5fecadea-8e8f-4531-a9c8-13e8f2c07c92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:43:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1a8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19190" uid: cbe264c0-6ad6-4eaf-b3bf-ab0908bae8ed kind: Event lastTimestamp: "2026-04-17T17:44:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:06Z" name: error-404-isvc-d1a8d-predictor.18a735e414ff524b namespace: kserve-ci-e2e-test resourceVersion: "19485" uid: 51d7ef4c-8765-4fd0-a331-129504e3415a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:44:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1a8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19364" uid: cbe264c0-6ad6-4eaf-b3bf-ab0908bae8ed kind: Event lastTimestamp: "2026-04-17T17:44:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:36Z" name: error-404-isvc-d1a8d-predictor.18a735eb12787a27 namespace: kserve-ci-e2e-test resourceVersion: "19889" uid: 0347c9c2-f8e5-488b-a847-e280315c2ac4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:44:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d1a8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19364" uid: cbe264c0-6ad6-4eaf-b3bf-ab0908bae8ed kind: Event lastTimestamp: "2026-04-17T17:44:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:44:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:36Z" name: error-404-isvc-d1a8d-predictor.18a735eb12db5773 namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: beebe720-6f85-4c75-b32b-667dbdbce4a2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1a8d namespace: kserve-ci-e2e-test resourceVersion: "19160" uid: 0575ac97-9c7e-402c-977a-7720b9237d99 kind: Event lastTimestamp: "2026-04-17T17:44:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:44:24Z" name: error-404-isvc-d1a8d.18a735e095da087e namespace: kserve-ci-e2e-test resourceVersion: "19638" uid: 08a71f8d-e032-4287-bddf-8998e61f4a3f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1a8d namespace: kserve-ci-e2e-test resourceVersion: "19160" uid: 0575ac97-9c7e-402c-977a-7720b9237d99 kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-d1a8d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d1a8d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:43:36Z" name: error-404-isvc-d1a8d.18a735e096692774 namespace: kserve-ci-e2e-test resourceVersion: "19196" uid: b5e3af03-adcb-4a7b-bfda-b54974920cf0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1a8d namespace: kserve-ci-e2e-test resourceVersion: "19160" uid: 0575ac97-9c7e-402c-977a-7720b9237d99 kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d1a8d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:43:36Z" name: error-404-isvc-d1a8d.18a735e0966958ba namespace: kserve-ci-e2e-test resourceVersion: "19197" uid: 04bca2a6-e152-4f7d-8c7a-0fdf4fb91074 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d1a8d namespace: kserve-ci-e2e-test resourceVersion: "19636" uid: 0575ac97-9c7e-402c-977a-7720b9237d99 kind: Event lastTimestamp: "2026-04-17T17:44:24Z" message: InferenceService [error-404-isvc-d1a8d] is Ready metadata: creationTimestamp: "2026-04-17T17:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:44:24Z" name: error-404-isvc-d1a8d.18a735ebbe8d50c1 namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: f305bf88-318a-4fc2-93d3-139a931cbe7a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:34:21.745504Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 3291e5f9-dbb1-4380-81de-6f318659f1c2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d987b-predictor-5666cb86b9-jz6hx to ip-10-0-130-19.ec2.internal metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:34:21Z" name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx.18a7355f5e8e8b02 namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: e108c029-4924-4301-b4e2-bad287483f5a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx namespace: kserve-ci-e2e-test resourceVersion: "14823" uid: 3291e5f9-dbb1-4380-81de-6f318659f1c2 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:34:22Z" name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx.18a7355f785a7d94 namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: 07e525d9-7a1e-4063-8769-1cc3e34a26ed reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 3291e5f9-dbb1-4380-81de-6f318659f1c2 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx.18a7355f79a57570 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: e70c59ab-8629-4a29-b871-922bc47fbf19 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 3291e5f9-dbb1-4380-81de-6f318659f1c2 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx.18a7355f7caa9aed namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 0b1e6c92-1bc1-4769-ab52-6cbd3bc6706e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 3291e5f9-dbb1-4380-81de-6f318659f1c2 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx.18a7355f7d1e4de5 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: ba36faeb-db93-45c7-9b92-1d0e55067b07 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 3291e5f9-dbb1-4380-81de-6f318659f1c2 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx.18a7355f7d29d941 namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: cc8478ab-48e5-4a76-9954-40611c3058aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 3291e5f9-dbb1-4380-81de-6f318659f1c2 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx.18a7355f80e72828 namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: ce80ead9-c8b2-4353-8581-de227d7df5e0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 3291e5f9-dbb1-4380-81de-6f318659f1c2 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx.18a7355f815f8cba namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: 03cdc62a-b3df-4a53-93a4-38c5bb8e57c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T17:34:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 3291e5f9-dbb1-4380-81de-6f318659f1c2 kind: Event lastTimestamp: "2026-04-17T17:43:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:34:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:39Z" name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx.18a7355fd0be9fcc namespace: kserve-ci-e2e-test resourceVersion: "19267" uid: 9912afef-0b3e-4a77-b8f6-320930431f23 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 3291e5f9-dbb1-4380-81de-6f318659f1c2 kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:36Z" name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx.18a735e093447646 namespace: kserve-ci-e2e-test resourceVersion: "19174" uid: 35cf4c04-d0ba-438e-b252-9320800bbe78 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 3291e5f9-dbb1-4380-81de-6f318659f1c2 kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:36Z" name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx.18a735e09344f193 namespace: kserve-ci-e2e-test resourceVersion: "19177" uid: 4ad80d48-9b1a-498e-897b-0a116d5a9a9a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 3291e5f9-dbb1-4380-81de-6f318659f1c2 kind: Event lastTimestamp: "2026-04-17T17:43:39Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:39Z" name: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx.18a735e1451bf847 namespace: kserve-ci-e2e-test resourceVersion: "19266" uid: fdd92270-4bde-4d4d-9300-4aba48fc9e7d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d987b-predictor-5666cb86b9 namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: dddbe376-e4ca-49bc-b38a-a0c3c29d21fb kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: 'Created pod: error-404-isvc-d987b-predictor-5666cb86b9-jz6hx' metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:21Z" name: error-404-isvc-d987b-predictor-5666cb86b9.18a7355f5e073ddf namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: f861fa9e-7182-4ee9-aaab-993e0c7a456e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d987b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14804" uid: 3aa82b21-e917-47a5-87a7-de3d92976877 kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: Scaled up replica set error-404-isvc-d987b-predictor-5666cb86b9 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:21Z" name: error-404-isvc-d987b-predictor.18a7355f5c669c22 namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: de017ca3-0e5e-4835-a6bd-a384b47e92b6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:34:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d987b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 5c0ffe11-5c28-48d7-8e23-81bdb05e6bdf kind: Event lastTimestamp: "2026-04-17T17:34:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:51Z" name: error-404-isvc-d987b-predictor.18a73562dabf969d namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: 006ce3d2-f7b2-433f-a588-65afac226e9c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:34:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d987b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 5c0ffe11-5c28-48d7-8e23-81bdb05e6bdf kind: Event lastTimestamp: "2026-04-17T17:34:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:51Z" name: error-404-isvc-d987b-predictor.18a73562db2a6131 namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: df408c5a-0cc3-44db-9a2b-bbd2ee0b4b61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:35:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d987b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: 5c0ffe11-5c28-48d7-8e23-81bdb05e6bdf kind: Event lastTimestamp: "2026-04-17T17:35:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:36Z" name: error-404-isvc-d987b-predictor.18a73569d8843284 namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: 32b3ecf6-737c-417c-8ae6-28485d230ed8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:35:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d987b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: 5c0ffe11-5c28-48d7-8e23-81bdb05e6bdf kind: Event lastTimestamp: "2026-04-17T17:35:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:36Z" name: error-404-isvc-d987b-predictor.18a73569d8f5bac7 namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: 12258f09-e949-4f76-b409-d3a863450a2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d987b namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: dc2f30fd-9906-4edb-beb3-430b79f9a1e3 kind: Event lastTimestamp: "2026-04-17T17:35:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:35:09Z" name: error-404-isvc-d987b.18a7355f5c15e7a6 namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: df70410d-f6e1-4f91-9186-28c6798687ff reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d987b namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: dc2f30fd-9906-4edb-beb3-430b79f9a1e3 kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: 'Failed to update status for InferenceService "error-404-isvc-d987b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d987b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:34:21Z" name: error-404-isvc-d987b.18a7355f5c7aacdb namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: 9d3093c7-6014-4694-b085-ad34df11f528 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d987b namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: dc2f30fd-9906-4edb-beb3-430b79f9a1e3 kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d987b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:34:21Z" name: error-404-isvc-d987b.18a7355f5c7af541 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 7103af66-566f-49bb-96f8-b4bf383c7668 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d987b namespace: kserve-ci-e2e-test resourceVersion: "15334" uid: dc2f30fd-9906-4edb-beb3-430b79f9a1e3 kind: Event lastTimestamp: "2026-04-17T17:35:09Z" message: InferenceService [error-404-isvc-d987b] is Ready metadata: creationTimestamp: "2026-04-17T17:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:35:09Z" name: error-404-isvc-d987b.18a7356a8bc407d8 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: fa61a661-7fc7-4a50-ba93-50aaa2f52732 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:45:36.639622Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4 namespace: kserve-ci-e2e-test resourceVersion: "20701" uid: 4664528d-dfcc-4452-8664-0c3273aca6cc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-dd512-predictor-844bd6fb77-gxzv4 to ip-10-0-130-19.ec2.internal metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:45:36Z" name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4.18a735fc816186cd namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 3bbe0f16-168f-4197-b24e-1c5b3b4d4639 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4 namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 4664528d-dfcc-4452-8664-0c3273aca6cc kind: Event lastTimestamp: "2026-04-17T17:45:37Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:45:37Z" name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4.18a735fc9a8a13f5 namespace: kserve-ci-e2e-test resourceVersion: "20727" uid: 8f6b0f4a-7311-43fd-ba25-9158c8215405 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 4664528d-dfcc-4452-8664-0c3273aca6cc kind: Event lastTimestamp: "2026-04-17T17:45:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:37Z" name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4.18a735fc9bd82843 namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: 2d115724-2cf4-4c74-b62d-ece0e2d4f128 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 4664528d-dfcc-4452-8664-0c3273aca6cc kind: Event lastTimestamp: "2026-04-17T17:45:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:37Z" name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4.18a735fc9ebf9eda namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: d3e8e102-3b71-4b24-9db0-9856330ded82 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 4664528d-dfcc-4452-8664-0c3273aca6cc kind: Event lastTimestamp: "2026-04-17T17:45:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:37Z" name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4.18a735fc9f46141e namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: 8062cdcc-2ea9-4499-8c6e-bdd16ad492b1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 4664528d-dfcc-4452-8664-0c3273aca6cc kind: Event lastTimestamp: "2026-04-17T17:45:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:37Z" name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4.18a735fc9f52cf3b namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: 87dc507a-08cf-44ac-adb3-f752a9306b43 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 4664528d-dfcc-4452-8664-0c3273aca6cc kind: Event lastTimestamp: "2026-04-17T17:45:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:37Z" name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4.18a735fca2439387 namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: 5b9bbbfc-9c35-4325-99fe-612404ac1d63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 4664528d-dfcc-4452-8664-0c3273aca6cc kind: Event lastTimestamp: "2026-04-17T17:45:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:37Z" name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4.18a735fca2bda617 namespace: kserve-ci-e2e-test resourceVersion: "20736" uid: cc3bc14f-9725-4f51-902e-0444a990b92e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:45:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 4664528d-dfcc-4452-8664-0c3273aca6cc kind: Event lastTimestamp: "2026-04-17T17:46:14Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:46:14Z" name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4.18a735fd0d425611 namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 33d133e3-c187-4ea8-a475-7c21e2cf8a6f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 4664528d-dfcc-4452-8664-0c3273aca6cc kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:51Z" name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4.18a7367daadee9a6 namespace: kserve-ci-e2e-test resourceVersion: "24961" uid: 6b9395da-7dec-4221-a444-688f1ac91751 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 4664528d-dfcc-4452-8664-0c3273aca6cc kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:51Z" name: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4.18a7367daadf8231 namespace: kserve-ci-e2e-test resourceVersion: "24963" uid: b0a2afd1-7265-4003-b4ff-042964239efc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-dd512-predictor-844bd6fb77 namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: 4a17db4d-d9da-451d-8264-1c15e0e3cdf8 kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: 'Created pod: error-404-isvc-dd512-predictor-844bd6fb77-gxzv4' metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:36Z" name: error-404-isvc-dd512-predictor-844bd6fb77.18a735fc80f23d5c namespace: kserve-ci-e2e-test resourceVersion: "20702" uid: 7afaba87-ab9f-4069-8c52-00ef4a1a44e6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-dd512-predictor namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: b405fce7-1002-4ea2-a424-092a2551f55e kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: Scaled up replica set error-404-isvc-dd512-predictor-844bd6fb77 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:36Z" name: error-404-isvc-dd512-predictor.18a735fc7f4c8457 namespace: kserve-ci-e2e-test resourceVersion: "20694" uid: f67ff3e7-9e0f-4679-806e-8c585fba058c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:45:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dd512-predictor namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: 5d89c4b8-5646-47ed-9b42-610a663e6edd kind: Event lastTimestamp: "2026-04-17T17:46:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:06Z" name: error-404-isvc-dd512-predictor.18a735fffe0f0490 namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: 8bd9d2fa-b853-4446-a7eb-41b7a8c56fb1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:45:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dd512-predictor namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: 5d89c4b8-5646-47ed-9b42-610a663e6edd kind: Event lastTimestamp: "2026-04-17T17:46:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:06Z" name: error-404-isvc-dd512-predictor.18a735fffe827787 namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: f336766f-2dff-482d-b2e3-6e7dbf8fa654 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:46:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dd512-predictor namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 5d89c4b8-5646-47ed-9b42-610a663e6edd kind: Event lastTimestamp: "2026-04-17T17:46:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:36Z" name: error-404-isvc-dd512-predictor.18a73606fc10aa4c namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: 1d65c687-5618-483c-9806-e376901e3191 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:46:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dd512-predictor namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 5d89c4b8-5646-47ed-9b42-610a663e6edd kind: Event lastTimestamp: "2026-04-17T17:46:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:36Z" name: error-404-isvc-dd512-predictor.18a73606fc7da2d1 namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 1b8dff2a-b72f-40bb-9889-c8988ffb4779 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dd512 namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 719db0a2-53bb-4528-aadd-ba85e9d9c6b9 kind: Event lastTimestamp: "2026-04-17T17:46:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:46:25Z" name: error-404-isvc-dd512.18a735fc7f50d6e1 namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 6b232012-84ed-48a6-9c90-2bdd5e8225e5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dd512 namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 719db0a2-53bb-4528-aadd-ba85e9d9c6b9 kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-dd512": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dd512": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:36Z" name: error-404-isvc-dd512.18a735fc7fda42c9 namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 0082af8a-09cc-4940-8a4c-6a1ceb3a9bc1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dd512 namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 719db0a2-53bb-4528-aadd-ba85e9d9c6b9 kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dd512": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:36Z" name: error-404-isvc-dd512.18a735fc7fda8399 namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: 1fdb2522-ac34-4b92-aa65-c4b046df6abb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:46:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dd512 namespace: kserve-ci-e2e-test resourceVersion: "21197" uid: 719db0a2-53bb-4528-aadd-ba85e9d9c6b9 kind: Event lastTimestamp: "2026-04-17T17:46:25Z" message: InferenceService [error-404-isvc-dd512] is Ready metadata: creationTimestamp: "2026-04-17T17:46:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:46:25Z" name: error-404-isvc-dd512.18a73607c87c5677 namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: e280d1c3-f4a5-4cb0-a0e0-44c2f11c2e5b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:54:01.904543Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg namespace: kserve-ci-e2e-test resourceVersion: "24441" uid: 8f5d59d1-3867-4c2a-b48c-5c7206893353 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e4a99-predictor-f849b4d65-x75zg to ip-10-0-130-19.ec2.internal metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:54:01Z" name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg.18a7367225845fce namespace: kserve-ci-e2e-test resourceVersion: "24445" uid: 2833d45e-7e25-4a2c-be2f-35690d5c9d6e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg namespace: kserve-ci-e2e-test resourceVersion: "24451" uid: 8f5d59d1-3867-4c2a-b48c-5c7206893353 kind: Event lastTimestamp: "2026-04-17T17:54:02Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:54:02Z" name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg.18a736723ec6c65b namespace: kserve-ci-e2e-test resourceVersion: "24464" uid: 5cbba9dd-ff07-406a-bb94-7a2696ef0810 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg namespace: kserve-ci-e2e-test resourceVersion: "24443" uid: 8f5d59d1-3867-4c2a-b48c-5c7206893353 kind: Event lastTimestamp: "2026-04-17T17:54:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:02Z" name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg.18a736724045b3cf namespace: kserve-ci-e2e-test resourceVersion: "24466" uid: 3125410a-398c-4576-9443-3ac1d2361643 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg namespace: kserve-ci-e2e-test resourceVersion: "24443" uid: 8f5d59d1-3867-4c2a-b48c-5c7206893353 kind: Event lastTimestamp: "2026-04-17T17:54:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:02Z" name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg.18a7367243c02aa9 namespace: kserve-ci-e2e-test resourceVersion: "24467" uid: 006036b9-ebf2-4ffc-a943-9d2f556d03df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg namespace: kserve-ci-e2e-test resourceVersion: "24443" uid: 8f5d59d1-3867-4c2a-b48c-5c7206893353 kind: Event lastTimestamp: "2026-04-17T17:54:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:02Z" name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg.18a736724436f610 namespace: kserve-ci-e2e-test resourceVersion: "24468" uid: 2f835a56-0440-4359-a186-2f05e42a3d39 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg namespace: kserve-ci-e2e-test resourceVersion: "24443" uid: 8f5d59d1-3867-4c2a-b48c-5c7206893353 kind: Event lastTimestamp: "2026-04-17T17:54:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:02Z" name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg.18a736724444d4ee namespace: kserve-ci-e2e-test resourceVersion: "24469" uid: 5965cbdd-72b8-473f-a5a6-3191acfd2304 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg namespace: kserve-ci-e2e-test resourceVersion: "24443" uid: 8f5d59d1-3867-4c2a-b48c-5c7206893353 kind: Event lastTimestamp: "2026-04-17T17:54:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:02Z" name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg.18a736724762de67 namespace: kserve-ci-e2e-test resourceVersion: "24470" uid: 3d728a07-2f9c-45a7-88cc-1ada79bdab7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg namespace: kserve-ci-e2e-test resourceVersion: "24443" uid: 8f5d59d1-3867-4c2a-b48c-5c7206893353 kind: Event lastTimestamp: "2026-04-17T17:54:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:02Z" name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg.18a73672487d3c6c namespace: kserve-ci-e2e-test resourceVersion: "24471" uid: b11d15e3-2538-4be0-b9e6-4eb7d89cc4c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:54:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg namespace: kserve-ci-e2e-test resourceVersion: "24443" uid: 8f5d59d1-3867-4c2a-b48c-5c7206893353 kind: Event lastTimestamp: "2026-04-17T17:54:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:39Z" name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg.18a73672907f2e57 namespace: kserve-ci-e2e-test resourceVersion: "24780" uid: 3071dd97-3ac0-4b91-97e4-0f1d45fdd19b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg namespace: kserve-ci-e2e-test resourceVersion: "24443" uid: 8f5d59d1-3867-4c2a-b48c-5c7206893353 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg.18a736828170cba0 namespace: kserve-ci-e2e-test resourceVersion: "25322" uid: 1c14bfe0-096e-41ec-91c2-4c87dda2a529 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg namespace: kserve-ci-e2e-test resourceVersion: "24443" uid: 8f5d59d1-3867-4c2a-b48c-5c7206893353 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg.18a7368281739726 namespace: kserve-ci-e2e-test resourceVersion: "25325" uid: 47487e1d-c607-41bd-9b34-41d83273cbec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg namespace: kserve-ci-e2e-test resourceVersion: "24443" uid: 8f5d59d1-3867-4c2a-b48c-5c7206893353 kind: Event lastTimestamp: "2026-04-17T17:55:14Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:14Z" name: error-404-isvc-e4a99-predictor-f849b4d65-x75zg.18a7368318873c9d namespace: kserve-ci-e2e-test resourceVersion: "25396" uid: af92b3c0-ee03-4eb7-94f8-7ca61b4d5142 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e4a99-predictor-f849b4d65 namespace: kserve-ci-e2e-test resourceVersion: "24430" uid: 126b50a1-3ad0-4496-9c9b-1f523228df38 kind: Event lastTimestamp: "2026-04-17T17:54:01Z" message: 'Created pod: error-404-isvc-e4a99-predictor-f849b4d65-x75zg' metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:54:01Z" name: error-404-isvc-e4a99-predictor-f849b4d65.18a73672251c4166 namespace: kserve-ci-e2e-test resourceVersion: "24442" uid: f1ee9c05-facf-42ab-8342-fe70b2979f3d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e4a99-predictor namespace: kserve-ci-e2e-test resourceVersion: "24427" uid: 7295f451-25a6-4720-b0f7-a1ea705054f3 kind: Event lastTimestamp: "2026-04-17T17:54:01Z" message: Scaled up replica set error-404-isvc-e4a99-predictor-f849b4d65 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:54:01Z" name: error-404-isvc-e4a99-predictor.18a736722350fff5 namespace: kserve-ci-e2e-test resourceVersion: "24433" uid: 6dcb1b00-b276-44df-a8b1-e8af25f49a29 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:54:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4a99-predictor namespace: kserve-ci-e2e-test resourceVersion: "24429" uid: 2e61bf2e-eb22-4faa-863a-53cb03f05321 kind: Event lastTimestamp: "2026-04-17T17:54:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:54:31Z" name: error-404-isvc-e4a99-predictor.18a73675a1c80685 namespace: kserve-ci-e2e-test resourceVersion: "24721" uid: 9393663e-2750-4417-9466-97028a384e69 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:54:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4a99-predictor namespace: kserve-ci-e2e-test resourceVersion: "24429" uid: 2e61bf2e-eb22-4faa-863a-53cb03f05321 kind: Event lastTimestamp: "2026-04-17T17:54:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:54:31Z" name: error-404-isvc-e4a99-predictor.18a73675a2314fcf namespace: kserve-ci-e2e-test resourceVersion: "24722" uid: 254f26dc-4e88-47dc-8d9e-8936b1c2f2d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:54:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4a99-predictor namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: 2e61bf2e-eb22-4faa-863a-53cb03f05321 kind: Event lastTimestamp: "2026-04-17T17:55:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:01Z" name: error-404-isvc-e4a99-predictor.18a7367c9faf8439 namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: 5fa34418-db32-4038-b572-fee496177287 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:54:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e4a99-predictor namespace: kserve-ci-e2e-test resourceVersion: "24605" uid: 2e61bf2e-eb22-4faa-863a-53cb03f05321 kind: Event lastTimestamp: "2026-04-17T17:55:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:01Z" name: error-404-isvc-e4a99-predictor.18a7367c9ff93f83 namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 07fd408d-50e9-4576-a7cc-0ef6ad445216 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4a99 namespace: kserve-ci-e2e-test resourceVersion: "24404" uid: 276651e4-ede5-4a8b-8754-1c7074cb3552 kind: Event lastTimestamp: "2026-04-17T17:54:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:54:49Z" name: error-404-isvc-e4a99.18a736722321ba9d namespace: kserve-ci-e2e-test resourceVersion: "24874" uid: 705c7d0c-7fbf-45b3-99b5-61dc3adcdd46 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4a99 namespace: kserve-ci-e2e-test resourceVersion: "24404" uid: 276651e4-ede5-4a8b-8754-1c7074cb3552 kind: Event lastTimestamp: "2026-04-17T17:54:01Z" message: 'Failed to update status for InferenceService "error-404-isvc-e4a99": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e4a99": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:54:01Z" name: error-404-isvc-e4a99.18a7367223972108 namespace: kserve-ci-e2e-test resourceVersion: "24436" uid: 6cad4909-57e1-4a84-b2d6-4bd5a3193df3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4a99 namespace: kserve-ci-e2e-test resourceVersion: "24404" uid: 276651e4-ede5-4a8b-8754-1c7074cb3552 kind: Event lastTimestamp: "2026-04-17T17:54:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e4a99": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:54:01Z" name: error-404-isvc-e4a99.18a73672239781bc namespace: kserve-ci-e2e-test resourceVersion: "24439" uid: e7dc185f-d9f1-4ef2-bbd4-3ecf4a9b8190 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e4a99 namespace: kserve-ci-e2e-test resourceVersion: "24872" uid: 276651e4-ede5-4a8b-8754-1c7074cb3552 kind: Event lastTimestamp: "2026-04-17T17:54:49Z" message: InferenceService [error-404-isvc-e4a99] is Ready metadata: creationTimestamp: "2026-04-17T17:54:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:54:49Z" name: error-404-isvc-e4a99.18a7367d4b6e21ae namespace: kserve-ci-e2e-test resourceVersion: "24873" uid: dea21b83-ce32-4d51-9d49-da38e4d3340c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:35:11.581558Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7 namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 53938468-83ca-43fd-8056-3e474d3956a9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fa93d-predictor-6ff7867c59-kgth7 to ip-10-0-130-19.ec2.internal metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:35:11Z" name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7.18a7356af9045a6c namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: e9449da6-1928-4471-bd39-84f1735567c7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 53938468-83ca-43fd-8056-3e474d3956a9 kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-fa93d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:11Z" name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7.18a7356b0423e3cf namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: 05953e2c-0436-4b6e-b8d0-994de740a047 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7 namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: 53938468-83ca-43fd-8056-3e474d3956a9 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:35:12Z" name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7.18a7356b3658e6db namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: f6f93bd9-901b-4668-a6e2-4e61867d167f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 53938468-83ca-43fd-8056-3e474d3956a9 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:12Z" name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7.18a7356b37c3860d namespace: kserve-ci-e2e-test resourceVersion: "15497" uid: 0bbd1e7e-3c61-47f0-98f7-c31c3cc7e44e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 53938468-83ca-43fd-8056-3e474d3956a9 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:12Z" name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7.18a7356b3ad3e844 namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 088d28a5-3aac-4c5a-94ea-f44bce7bf9e6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 53938468-83ca-43fd-8056-3e474d3956a9 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:12Z" name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7.18a7356b3b462977 namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: 9c982131-af1d-4210-8f61-6eeff1b7645e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 53938468-83ca-43fd-8056-3e474d3956a9 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:12Z" name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7.18a7356b3b528ad8 namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: 897acaa9-b959-46c3-99f5-35faa0a9d5f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 53938468-83ca-43fd-8056-3e474d3956a9 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:12Z" name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7.18a7356b3f526b19 namespace: kserve-ci-e2e-test resourceVersion: "15506" uid: f6ca4fa1-fce9-480e-9a77-c44eac024479 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 53938468-83ca-43fd-8056-3e474d3956a9 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:12Z" name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7.18a7356b3fc74347 namespace: kserve-ci-e2e-test resourceVersion: "15507" uid: 977e1ff8-e738-4156-8635-66cfe40ca6a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 53938468-83ca-43fd-8056-3e474d3956a9 kind: Event lastTimestamp: "2026-04-17T17:35:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:49Z" name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7.18a7356b7fdc164d namespace: kserve-ci-e2e-test resourceVersion: "15864" uid: 7ff3a38e-b6b7-4b12-b165-1568d1be1d98 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 53938468-83ca-43fd-8056-3e474d3956a9 kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:26Z" name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7.18a735ec26646420 namespace: kserve-ci-e2e-test resourceVersion: "19729" uid: fde445f7-78bb-44dc-b419-826ecbbd4d6f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7 namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: 53938468-83ca-43fd-8056-3e474d3956a9 kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:26Z" name: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7.18a735ec2664dfbf namespace: kserve-ci-e2e-test resourceVersion: "19732" uid: ca7a3f7d-7f34-49d1-be9c-ed8ef30630ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fa93d-predictor-6ff7867c59 namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: 5affd437-f781-4d66-809e-1c85d5d410d0 kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: 'Created pod: error-404-isvc-fa93d-predictor-6ff7867c59-kgth7' metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:11Z" name: error-404-isvc-fa93d-predictor-6ff7867c59.18a7356af87d1365 namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: e0196e6e-af15-4730-9708-c7658a40d3b8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fa93d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: 48e008a2-6151-4e6e-889f-48deb9ef8292 kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: Scaled up replica set error-404-isvc-fa93d-predictor-6ff7867c59 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:11Z" name: error-404-isvc-fa93d-predictor.18a7356af6d1ff34 namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: f835f47e-1d2d-4df0-915d-eec30d4aa670 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:35:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fa93d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15436" uid: 6bfb815f-3e5d-4fce-a435-be061867f6bc kind: Event lastTimestamp: "2026-04-17T17:35:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:56Z" name: error-404-isvc-fa93d-predictor.18a7356e756cd372 namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: 8c50497f-9df5-4ed2-aa28-de1f0e6a274e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:35:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fa93d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15436" uid: 6bfb815f-3e5d-4fce-a435-be061867f6bc kind: Event lastTimestamp: "2026-04-17T17:35:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:56Z" name: error-404-isvc-fa93d-predictor.18a7356e76362c4b namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: adbb164e-bd86-4a73-ad5f-b20b4b0a7c21 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fa93d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15682" uid: 6bfb815f-3e5d-4fce-a435-be061867f6bc kind: Event lastTimestamp: "2026-04-17T17:36:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:36:26Z" name: error-404-isvc-fa93d-predictor.18a73578f220342e namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 8b37d4bb-17af-40c6-a100-20677d52c6f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fa93d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15682" uid: 6bfb815f-3e5d-4fce-a435-be061867f6bc kind: Event lastTimestamp: "2026-04-17T17:36:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:36:26Z" name: error-404-isvc-fa93d-predictor.18a73578f2a0dd0a namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: 44a9107b-44c7-460e-869e-a0aa5cc6594b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fa93d namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 8f370b1b-d3b2-4660-8267-4b404e7d5b82 kind: Event lastTimestamp: "2026-04-17T17:35:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:35:59Z" name: error-404-isvc-fa93d.18a7356af6a49685 namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 8f1b0710-3359-44d8-a126-31f024b79612 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fa93d namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 8f370b1b-d3b2-4660-8267-4b404e7d5b82 kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-fa93d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fa93d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:35:11Z" name: error-404-isvc-fa93d.18a7356af71f151a namespace: kserve-ci-e2e-test resourceVersion: "15442" uid: 46475023-3b97-4d47-a61a-2a275384a48d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fa93d namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: 8f370b1b-d3b2-4660-8267-4b404e7d5b82 kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fa93d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:35:11Z" name: error-404-isvc-fa93d.18a7356af71f6967 namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: e74d45a3-ec28-4696-9b45-08b5fefcf613 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fa93d namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: 8f370b1b-d3b2-4660-8267-4b404e7d5b82 kind: Event lastTimestamp: "2026-04-17T17:35:59Z" message: InferenceService [error-404-isvc-fa93d] is Ready metadata: creationTimestamp: "2026-04-17T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:35:59Z" name: error-404-isvc-fa93d.18a735763ad2f4cb namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: 0867088d-46cc-4f43-8d40-81e174e634e1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:32:51.264819Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13685" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-5b497dcd98-97286 to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:32:51Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7354a4d7cd051 namespace: kserve-ci-e2e-test resourceVersion: "13692" uid: f968eab6-f938-46c2-aae5-4c1af1e4ef5b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13696" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:32:51Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7354a67405daa namespace: kserve-ci-e2e-test resourceVersion: "13767" uid: 540e0b83-9ae1-4d70-a78f-f2ab5337095f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:17Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:4cb151dc63436df7d13d4e5ee0cc2d7fdd1ff90cc80417c51a70ddeb283661c4" metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:17Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7354a68a15cd7 namespace: kserve-ci-e2e-test resourceVersion: "14107" uid: 8aeed9ca-9437-4999-b447-7f894e792bb9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:02Z" message: 'Failed to pull image "quay.io/opendatahub/kserve-storage-initializer@sha256:4cb151dc63436df7d13d4e5ee0cc2d7fdd1ff90cc80417c51a70ddeb283661c4": unable to pull image or OCI artifact: pull image err: copying system image from manifest list: parsing image configuration: fetching blob: received unexpected HTTP status: 504 Gateway Time-out; artifact err: provided artifact is a container image' metadata: creationTimestamp: "2026-04-17T17:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:02Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7354cd75c6c6a namespace: kserve-ci-e2e-test resourceVersion: "13933" uid: daff464b-02e8-4126-a225-f4559e377c9a reason: Failed reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:02Z" message: 'Error: ErrImagePull' metadata: creationTimestamp: "2026-04-17T17:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:02Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7354cd75ceb09 namespace: kserve-ci-e2e-test resourceVersion: "13934" uid: 79bf4e8b-e0e5-469a-8edf-f47fe768888d reason: Failed reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:02Z" message: Back-off pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:4cb151dc63436df7d13d4e5ee0cc2d7fdd1ff90cc80417c51a70ddeb283661c4" metadata: creationTimestamp: "2026-04-17T17:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:02Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7354d05080bc1 namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 716399b3-bc8f-495a-9604-4d28bdca950a reason: BackOff reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:02Z" message: 'Error: ImagePullBackOff' metadata: creationTimestamp: "2026-04-17T17:33:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:02Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7354d0508588f namespace: kserve-ci-e2e-test resourceVersion: "13943" uid: fda4f801-17cb-43d0-be67-9a14d604b05c reason: Failed reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:19Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:4cb151dc63436df7d13d4e5ee0cc2d7fdd1ff90cc80417c51a70ddeb283661c4" in 2.837s (2.837s including waiting). Image size: 300962214 bytes.' metadata: creationTimestamp: "2026-04-17T17:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:19Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a73550f9eeb095 namespace: kserve-ci-e2e-test resourceVersion: "14128" uid: bd3c2460-83ef-4553-9f52-90651fcc43b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:19Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T17:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:19Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a73550fcc2473e namespace: kserve-ci-e2e-test resourceVersion: "14130" uid: d1401516-b881-4bf5-b608-11b74bf637c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:19Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:19Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T17:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:19Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a73550fd4942ee namespace: kserve-ci-e2e-test resourceVersion: "14131" uid: fcf4fbd6-794e-4f9d-a855-9953bb18ea25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:24Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" metadata: creationTimestamp: "2026-04-17T17:33:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:24Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a73551edeb4143 namespace: kserve-ci-e2e-test resourceVersion: "14178" uid: 8efb4d91-772e-4047-9c39-d9f8126cf75d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:30Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" in 6.41s (6.41s including waiting). Image size: 1560702890 bytes.' metadata: creationTimestamp: "2026-04-17T17:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:30Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a735536c03ad24 namespace: kserve-ci-e2e-test resourceVersion: "14239" uid: eff7ea80-b66d-4297-9446-e61adc5788b2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:30Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7355373165ffe namespace: kserve-ci-e2e-test resourceVersion: "14242" uid: 67c461e4-e7ce-4d8b-8ff0-1a53b0da5f20 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:30Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a735537449bed9 namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 7cf37355-c599-4c4a-b381-364709f7a3a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:30Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a735537455cdc6 namespace: kserve-ci-e2e-test resourceVersion: "14244" uid: e5387800-865e-423c-afec-30a77c2a1519 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:30Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7355377ce0088 namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: 5b3ba774-ca35-473d-9144-da4ca3d0779c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:33:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:30Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7355378562e68 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: d3d32569-54af-4ce3-b79b-b31657056d97 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-17T17:33:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:34:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:33:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:27Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7355390d5d400 namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: c44597e4-9270-416f-b0dc-f6f979ba32f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:11Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7356af128c993 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 348bd238-33fb-4fc5-8b5d-9b80b5c4c42d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:11Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7356af1297994 namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: 2fad5bfc-3767-4a1d-877b-da448cd1fa13 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286 namespace: kserve-ci-e2e-test resourceVersion: "13688" uid: 5c934c46-580b-4625-b33b-262e329e8b7a kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: 'Readiness probe failed: Get "https://10.132.0.30:8643/healthz": dial tcp 10.132.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:12Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-97286.18a7356b1508759d namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: ed2ab02f-90f5-45cb-aa88-80fd70488ceb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-5b497dcd98 namespace: kserve-ci-e2e-test resourceVersion: "13672" uid: e0df469c-3639-4387-9a05-a920770fe2ea kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-5b497dcd98-97286' metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:32:51Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98.18a7354a4d06b7f5 namespace: kserve-ci-e2e-test resourceVersion: "13687" uid: e0f8eab7-9d3b-4913-8b6e-485ead961901 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13671" uid: 13a271d8-fcdf-4d38-b892-1034fea85405 kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-5b497dcd98 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:32:51Z" name: isvc-sklearn-graph-1-predictor.18a7354a46219e3c namespace: kserve-ci-e2e-test resourceVersion: "13673" uid: bcfca41e-0a44-4b36-9316-bd75f418298b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13683" uid: 6e0fae84-1e22-4c16-8706-3c94cb40db2e kind: Event lastTimestamp: "2026-04-17T17:34:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:06Z" name: isvc-sklearn-graph-1-predictor.18a7354dcc2ca3d7 namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: a99dcb56-34c6-4e54-951f-1cf8b5ed4e46 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13683" uid: 6e0fae84-1e22-4c16-8706-3c94cb40db2e kind: Event lastTimestamp: "2026-04-17T17:34:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:06Z" name: isvc-sklearn-graph-1-predictor.18a7354dcca4ca0a namespace: kserve-ci-e2e-test resourceVersion: "14550" uid: 28adbcf2-c8c2-4a88-bf34-21fe153a3490 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 6e0fae84-1e22-4c16-8706-3c94cb40db2e kind: Event lastTimestamp: "2026-04-17T17:35:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:06Z" name: isvc-sklearn-graph-1-predictor.18a735584a70ff4e namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: c61cf79a-1d07-4961-bb47-cbb3056e224d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 6e0fae84-1e22-4c16-8706-3c94cb40db2e kind: Event lastTimestamp: "2026-04-17T17:35:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:06Z" name: isvc-sklearn-graph-1-predictor.18a735584adee807 namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: c5078124-5cc9-4dd2-bdc0-87ced82e0b5e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 12 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13656" uid: 5ba82904-d13b-4512-b475-ee7f4e7cb03f kind: Event lastTimestamp: "2026-04-17T17:34:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:34:37Z" name: isvc-sklearn-graph-1.18a7354a4c59b127 namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: 70355621-4fbd-418d-85d4-629a934ae5e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13656" uid: 5ba82904-d13b-4512-b475-ee7f4e7cb03f kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: 'Failed to update status for InferenceService "isvc-sklearn-graph-1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:32:51Z" name: isvc-sklearn-graph-1.18a7354a4cde67da namespace: kserve-ci-e2e-test resourceVersion: "13686" uid: dde6dc9e-f218-4989-b353-dc97c00f7fba reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13656" uid: 5ba82904-d13b-4512-b475-ee7f4e7cb03f kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "isvc-sklearn-graph-1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:32:51Z" name: isvc-sklearn-graph-1.18a7354a4cdea844 namespace: kserve-ci-e2e-test resourceVersion: "13690" uid: 774780ef-d2d1-4cf5-bb43-66c091157c54 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: 5ba82904-d13b-4512-b475-ee7f4e7cb03f kind: Event lastTimestamp: "2026-04-17T17:34:37Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-17T17:34:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:34:37Z" name: isvc-sklearn-graph-1.18a73562f3dcb501 namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: 344bc327-4203-4377-bdef-e577836572cd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:32:52.466555Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13812" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq to ip-10-0-130-19.ec2.internal metadata: creationTimestamp: "2026-04-17T17:32:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:32:52Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354a951dd94b namespace: kserve-ci-e2e-test resourceVersion: "13816" uid: 73ffb72e-74b0-4c91-87d4-f125af840819 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13820" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:32:52Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:32:52Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354aae31e053 namespace: kserve-ci-e2e-test resourceVersion: "13831" uid: 0458bfd7-f908-48c1-9899-ffcb71dc0960 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:32:52Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:4cb151dc63436df7d13d4e5ee0cc2d7fdd1ff90cc80417c51a70ddeb283661c4" metadata: creationTimestamp: "2026-04-17T17:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:32:52Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354aaf91974b namespace: kserve-ci-e2e-test resourceVersion: "13834" uid: 1e513eb0-4415-47df-a533-f90609fdee68 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:32:58Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:4cb151dc63436df7d13d4e5ee0cc2d7fdd1ff90cc80417c51a70ddeb283661c4" in 5.103s (5.104s including waiting). Image size: 300962214 bytes.' metadata: creationTimestamp: "2026-04-17T17:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:32:58Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354bdfcb1777 namespace: kserve-ci-e2e-test resourceVersion: "13888" uid: 12949672-2778-465d-98e0-9b362aaeb270 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:32:58Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T17:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:32:58Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354be476504e namespace: kserve-ci-e2e-test resourceVersion: "13890" uid: 14f6c55b-516e-4ae2-91d2-1b86726710c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:32:58Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T17:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:32:58Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354be50ec4c6 namespace: kserve-ci-e2e-test resourceVersion: "13893" uid: f375c0ec-ece5-4de4-bfc7-5fe6ec09ac64 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:33:04Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" metadata: creationTimestamp: "2026-04-17T17:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:04Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354d5a8e67e9 namespace: kserve-ci-e2e-test resourceVersion: "13964" uid: a08cd00e-92e0-4154-ad10-4537e7dc61d3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:33:12Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" in 8.068s (8.068s including waiting). Image size: 1560702890 bytes.' metadata: creationTimestamp: "2026-04-17T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:12Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354f3b757440 namespace: kserve-ci-e2e-test resourceVersion: "14059" uid: 25f64616-2237-41d7-a00c-f1ab9cade07f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:33:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:12Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354f3f89b290 namespace: kserve-ci-e2e-test resourceVersion: "14063" uid: 47f9f70e-cdd6-4269-8923-29294787669d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:33:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:12Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354f40814407 namespace: kserve-ci-e2e-test resourceVersion: "14065" uid: d480e48e-ac70-4ce5-8d9c-2e2a784ccb4a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:33:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:12Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354f409e548e namespace: kserve-ci-e2e-test resourceVersion: "14066" uid: fe9b7258-95e7-49ce-b6f1-d01cc8b9437b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:33:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:12Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354f456b0a7b namespace: kserve-ci-e2e-test resourceVersion: "14067" uid: 69e93b68-7e17-48fc-b7d2-9c4a412769a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:33:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:33:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:12Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354f46133522 namespace: kserve-ci-e2e-test resourceVersion: "14068" uid: 4a4e13b2-7752-4762-9c63-24bf155e6288 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T17:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:34:20Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:20Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7354fb18fb612 namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 130d90e6-f647-4458-bd68-ece7b96614ef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:11Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7356af9f5a035 namespace: kserve-ci-e2e-test resourceVersion: "15465" uid: 7fcbc860-4517-489e-a772-8d7cc116c0ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:11Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7356af9f5ebf9 namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: dfe8d7eb-c983-4a8e-95d0-2b050fda5943 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: bf2da75f-8df0-44f2-8533-584f97edbb63 kind: Event lastTimestamp: "2026-04-17T17:35:15Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:35:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:15Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq.18a7356bddf33fa4 namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: 41facdf7-bf02-435b-a747-091adb65b697 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-847f6cf74b namespace: kserve-ci-e2e-test resourceVersion: "13799" uid: 4d10ced3-b669-48ad-8ae0-a26b3a6825a3 kind: Event lastTimestamp: "2026-04-17T17:32:52Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-847f6cf74b-x2rsq' metadata: creationTimestamp: "2026-04-17T17:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:32:52Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b.18a7354a94b22906 namespace: kserve-ci-e2e-test resourceVersion: "13813" uid: 2a5ff6ee-1aad-4685-b1e2-6b6b4626b242 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13797" uid: 72c4e678-2c31-40a0-8089-72e327d1ac54 kind: Event lastTimestamp: "2026-04-17T17:32:52Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-847f6cf74b from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:32:52Z" name: isvc-sklearn-graph-2-predictor.18a7354a7e7bd6aa namespace: kserve-ci-e2e-test resourceVersion: "13800" uid: ee6d1ea3-6e5e-4e3e-8f8f-144f5e2a3de6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:33:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13798" uid: 2f27065d-e65e-4741-88a0-40c8e28d6990 kind: Event lastTimestamp: "2026-04-17T17:33:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:33:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:33:22Z" name: isvc-sklearn-graph-2-predictor.18a7354dfd2a840d namespace: kserve-ci-e2e-test resourceVersion: "14164" uid: 7eaf3d3e-01c8-49e1-bec8-38bf9ed6bea7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:33:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13798" uid: 2f27065d-e65e-4741-88a0-40c8e28d6990 kind: Event lastTimestamp: "2026-04-17T17:33:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:33:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:33:22Z" name: isvc-sklearn-graph-2-predictor.18a7354dfd962277 namespace: kserve-ci-e2e-test resourceVersion: "14165" uid: 103e59b2-f054-494f-86dd-b7c87ee8196a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:33:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: 2f27065d-e65e-4741-88a0-40c8e28d6990 kind: Event lastTimestamp: "2026-04-17T17:34:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:52Z" name: isvc-sklearn-graph-2-predictor.18a73554fb48d368 namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 930397ac-d356-4fcf-acdf-4e6a83fad7b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:33:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: 2f27065d-e65e-4741-88a0-40c8e28d6990 kind: Event lastTimestamp: "2026-04-17T17:34:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:33:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:52Z" name: isvc-sklearn-graph-2-predictor.18a73554fbb965b6 namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: dce5226c-3512-4c58-94ac-44d3f2ae8ed3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T17:32:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13780" uid: 43bdbd9e-d059-46d1-ac0d-5de67c74679a kind: Event lastTimestamp: "2026-04-17T17:34:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:34:30Z" name: isvc-sklearn-graph-2.18a7354a7e52109a namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 51607ef2-bb2f-4b77-b282-ec4eb0ba8dc3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 43bdbd9e-d059-46d1-ac0d-5de67c74679a kind: Event lastTimestamp: "2026-04-17T17:34:30Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-17T17:34:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:34:30Z" name: isvc-sklearn-graph-2.18a7356168b9dece namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: c0e8cb47-b61c-45e2-acf1-d199dba4b242 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:32:51.574070Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-jpmw2 to ip-10-0-130-19.ec2.internal metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:32:51Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a7354a5febb324 namespace: kserve-ci-e2e-test resourceVersion: "13748" uid: 7c8f9d22-4e5a-4276-837f-f3f73d352c09 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:32:51Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a7354a687b8cd3 namespace: kserve-ci-e2e-test resourceVersion: "13770" uid: 54582c03-01c2-418b-832d-106d3afe18bf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13753" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:32:52Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:32:52Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a7354a9ca7883c namespace: kserve-ci-e2e-test resourceVersion: "13823" uid: d95cc8f2-178f-4450-bc22-d3704547c04a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:32:52Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:4cb151dc63436df7d13d4e5ee0cc2d7fdd1ff90cc80417c51a70ddeb283661c4" metadata: creationTimestamp: "2026-04-17T17:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:32:52Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a7354a9e07e363 namespace: kserve-ci-e2e-test resourceVersion: "13826" uid: b995d398-e405-4278-bb43-fe6f1b129348 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:57Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:32:57Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:4cb151dc63436df7d13d4e5ee0cc2d7fdd1ff90cc80417c51a70ddeb283661c4" in 5.378s (5.378s including waiting). Image size: 300962214 bytes.' metadata: creationTimestamp: "2026-04-17T17:32:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:32:57Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a7354bdea53476 namespace: kserve-ci-e2e-test resourceVersion: "13887" uid: 45b88099-89b4-4b47-860a-7157cfc8d66f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:32:58Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-17T17:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:32:58Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a7354be422358f namespace: kserve-ci-e2e-test resourceVersion: "13889" uid: 9308fb8e-e2b5-4a2c-be95-f5fb715cc9ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:32:58Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-17T17:32:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:32:58Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a7354be4aa7547 namespace: kserve-ci-e2e-test resourceVersion: "13891" uid: 8c1c2e84-957d-4793-b75e-9c2f0f9180f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:33:04Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-17T17:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:04Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a7354d5a7bb6a9 namespace: kserve-ci-e2e-test resourceVersion: "13963" uid: 45e504de-0269-45d0-8f2a-3db46599a85a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:33:26Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 22.568s (22.568s including waiting). Image size: 1306229499 bytes.' metadata: creationTimestamp: "2026-04-17T17:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:26Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a735529bae67b3 namespace: kserve-ci-e2e-test resourceVersion: "14203" uid: b95622c5-6f5e-4931-b271-9b8bf9e875ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:33:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:33:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a73552a16b5f37 namespace: kserve-ci-e2e-test resourceVersion: "14204" uid: ad5bcc25-6f2d-42c7-bafc-338141d40435 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:33:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:33:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a73552a2359407 namespace: kserve-ci-e2e-test resourceVersion: "14205" uid: ed793f0f-2113-4e6f-ac9a-3049f33e0a8b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:33:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:33:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a73552a24c81ea namespace: kserve-ci-e2e-test resourceVersion: "14206" uid: 2997b20b-92b8-46a9-a87a-a06d22e46461 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:33:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:33:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a73552a80eacfd namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 817c371f-ffd6-44f8-9731-fe731a87e0d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:33:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:33:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:27Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a73552a91c4c77 namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: db3f527a-1cef-4911-94b7-364daeed9cd7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T17:33:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:35:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:33:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:13Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a73552bbaff607 namespace: kserve-ci-e2e-test resourceVersion: "15521" uid: ab25bd29-7622-4d1f-b50f-af9b0df8cf12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:11Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a7356af57dbcf0 namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: 144b6c8d-b02c-4cd7-8b14-c1058692f27b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:11Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a7356af57e679c namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 898b2ce1-14f0-4341-81be-17492dadedc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: f19b3998-85b3-40b4-89f4-c66414780c36 kind: Event lastTimestamp: "2026-04-17T17:35:13Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:13Z" name: isvc-xgboost-graph-predictor-669d8d6456-jpmw2.18a7356b69f5bf89 namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 9382c5ed-657b-4d61-840b-12cb8b98a318 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-19.ec2.internal source: component: kubelet host: ip-10-0-130-19.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "13737" uid: d7d56325-0143-4e36-b306-7c5a398da94d kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-jpmw2' metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:32:51Z" name: isvc-xgboost-graph-predictor-669d8d6456.18a7354a5f7c1bae namespace: kserve-ci-e2e-test resourceVersion: "13746" uid: a5e4069f-21f4-4bf6-8805-11015e3d1ef5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13735" uid: d8fb19f3-c8b4-44f1-be5e-3698f336ee79 kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:32:51Z" name: isvc-xgboost-graph-predictor.18a7354a5c89f278 namespace: kserve-ci-e2e-test resourceVersion: "13741" uid: 703d2258-0e39-4fc2-b6d1-d49e0b5a928d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13736" uid: c1cd2aa1-47db-4acf-9565-8e47d236ca6b kind: Event lastTimestamp: "2026-04-17T17:33:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:33:51Z" name: isvc-xgboost-graph-predictor.18a7354ddae0710b namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 523c2e4e-be4e-4170-bdb0-f41623cf2698 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13736" uid: c1cd2aa1-47db-4acf-9565-8e47d236ca6b kind: Event lastTimestamp: "2026-04-17T17:33:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:33:51Z" name: isvc-xgboost-graph-predictor.18a7354ddb6e7d61 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 26baf92b-883a-4712-8872-0fa347894c41 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T17:34:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14009" uid: c1cd2aa1-47db-4acf-9565-8e47d236ca6b kind: Event lastTimestamp: "2026-04-17T17:34:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:51Z" name: isvc-xgboost-graph-predictor.18a7355bd6c23dde namespace: kserve-ci-e2e-test resourceVersion: "15136" uid: 8bf832dc-7992-4c4c-823a-15ad988c792d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T17:34:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14009" uid: c1cd2aa1-47db-4acf-9565-8e47d236ca6b kind: Event lastTimestamp: "2026-04-17T17:34:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:34:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:51Z" name: isvc-xgboost-graph-predictor.18a7355bd751ba12 namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: fdf2bfb0-0208-4a38-89a3-b40ed6f16e25 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13719" uid: b152c008-9f54-42fb-82db-8b24a1f96ac8 kind: Event lastTimestamp: "2026-04-17T17:34:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:34:33Z" name: isvc-xgboost-graph.18a7354a5c2cc5fa namespace: kserve-ci-e2e-test resourceVersion: "14977" uid: ef77f8cc-b76b-4991-9d84-1204626b1c34 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14975" uid: b152c008-9f54-42fb-82db-8b24a1f96ac8 kind: Event lastTimestamp: "2026-04-17T17:34:33Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-17T17:34:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:34:33Z" name: isvc-xgboost-graph.18a735621f511995 namespace: kserve-ci-e2e-test resourceVersion: "14976" uid: 4b3232f8-96a6-440b-b2d4-a66727400e0d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:35:01.222852Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-579966fbd6-9m9hb namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: 96980502-64fc-41d2-9ae9-1e7eefe8f6d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-579966fbd6-9m9hb to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:35:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:35:01Z" name: model-chainer-579966fbd6-9m9hb.18a735688f971f43 namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: 6fe82269-85be-4404-bebb-b16db7a8ffec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:01Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-579966fbd6-9m9hb namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: 96980502-64fc-41d2-9ae9-1e7eefe8f6d5 kind: Event lastTimestamp: "2026-04-17T17:35:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:35:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:01Z" name: model-chainer-579966fbd6-9m9hb.18a7356899466c14 namespace: kserve-ci-e2e-test resourceVersion: "15238" uid: f01ec5e4-6984-4b4c-aae7-d5640dfabeaf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:02Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-579966fbd6-9m9hb namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: 96980502-64fc-41d2-9ae9-1e7eefe8f6d5 kind: Event lastTimestamp: "2026-04-17T17:35:02Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:35:02Z" name: model-chainer-579966fbd6-9m9hb.18a73568ccc7f24d namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: 70fab5ab-621a-436c-bed8-a4a3570dc556 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-579966fbd6-9m9hb namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: 96980502-64fc-41d2-9ae9-1e7eefe8f6d5 kind: Event lastTimestamp: "2026-04-17T17:35:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0589637b57a3315022b8ebf7fc1b0f9831e3a45e07a3fc2fab24137dd8f190f9" already present on machine metadata: creationTimestamp: "2026-04-17T17:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:02Z" name: model-chainer-579966fbd6-9m9hb.18a73568ce2c7bc9 namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: b3c423c2-34d5-4bb2-bcd5-c6f7f27faf43 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-579966fbd6-9m9hb namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: 96980502-64fc-41d2-9ae9-1e7eefe8f6d5 kind: Event lastTimestamp: "2026-04-17T17:35:02Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-17T17:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:02Z" name: model-chainer-579966fbd6-9m9hb.18a73568d12f0c8e namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: b77793f3-b6e3-4b24-94d9-ee26e32d8334 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-579966fbd6-9m9hb namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: 96980502-64fc-41d2-9ae9-1e7eefe8f6d5 kind: Event lastTimestamp: "2026-04-17T17:35:02Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-17T17:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:02Z" name: model-chainer-579966fbd6-9m9hb.18a73568d1a01cd5 namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: c079b5c6-56e8-4236-ad3a-153edb7182be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-579966fbd6-9m9hb namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: 96980502-64fc-41d2-9ae9-1e7eefe8f6d5 kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:11Z" name: model-chainer-579966fbd6-9m9hb.18a7356ae99db438 namespace: kserve-ci-e2e-test resourceVersion: "15365" uid: f44a81e7-df88-4656-bbc8-a0239fe7af26 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:35:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-579966fbd6-9m9hb namespace: kserve-ci-e2e-test resourceVersion: "15224" uid: 96980502-64fc-41d2-9ae9-1e7eefe8f6d5 kind: Event lastTimestamp: "2026-04-17T17:35:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:35:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:38Z" name: model-chainer-579966fbd6-9m9hb.18a7356b63b3418b namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: 80f45b1c-6c37-4c43-abb9-ceee64cd7fa3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-579966fbd6 namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: b729868c-45be-4e6a-b435-b5cf26d4ad26 kind: Event lastTimestamp: "2026-04-17T17:35:01Z" message: 'Created pod: model-chainer-579966fbd6-9m9hb' metadata: creationTimestamp: "2026-04-17T17:35:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:01Z" name: model-chainer-579966fbd6.18a735688eff1adc namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 9d477947-9814-484c-bb66-7ea6bf985aa4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: 3a80e36f-cc1b-49c9-9746-f4e21eb79467 kind: Event lastTimestamp: "2026-04-17T17:35:01Z" message: Scaled up replica set model-chainer-579966fbd6 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:35:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:01Z" name: model-chainer.18a735688e096c2e namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: 813ebc07-271b-4653-a789-b4a50f8c37ed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15233" uid: 65089b30-a5f0-46b0-8bd9-769538a2d9db kind: Event lastTimestamp: "2026-04-17T17:35:01Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-17T17:35:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:35:01Z" name: model-chainer.18a73568936365d7 namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: 31f794f5-a533-4271-a622-9b6162dc2623 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:45:26.344717Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a1277-6d9f5c6dd8-sqgk9 namespace: kserve-ci-e2e-test resourceVersion: "20482" uid: 53213afd-bfbe-44e9-a0ce-5136a80871b2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a1277-6d9f5c6dd8-sqgk9 to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:45:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:45:26Z" name: sequence-graph-a1277-6d9f5c6dd8-sqgk9.18a735fa1bc1b108 namespace: kserve-ci-e2e-test resourceVersion: "20488" uid: 2dd059b0-6e03-4367-8903-8f768fc0f049 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:26Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a1277-6d9f5c6dd8-sqgk9 namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: 53213afd-bfbe-44e9-a0ce-5136a80871b2 kind: Event lastTimestamp: "2026-04-17T17:45:26Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:45:26Z" name: sequence-graph-a1277-6d9f5c6dd8-sqgk9.18a735fa350bb676 namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 702fe154-df4d-4411-9e85-d6737ce3fbdf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a1277} kind: Pod name: sequence-graph-a1277-6d9f5c6dd8-sqgk9 namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: 53213afd-bfbe-44e9-a0ce-5136a80871b2 kind: Event lastTimestamp: "2026-04-17T17:45:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0589637b57a3315022b8ebf7fc1b0f9831e3a45e07a3fc2fab24137dd8f190f9" already present on machine metadata: creationTimestamp: "2026-04-17T17:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:26Z" name: sequence-graph-a1277-6d9f5c6dd8-sqgk9.18a735fa364c1fb5 namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: e3de34e8-5c50-4685-83fe-b22bf3a71d88 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a1277} kind: Pod name: sequence-graph-a1277-6d9f5c6dd8-sqgk9 namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: 53213afd-bfbe-44e9-a0ce-5136a80871b2 kind: Event lastTimestamp: "2026-04-17T17:45:26Z" message: 'Created container: sequence-graph-a1277' metadata: creationTimestamp: "2026-04-17T17:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:26Z" name: sequence-graph-a1277-6d9f5c6dd8-sqgk9.18a735fa390df162 namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: c6ead3fc-a5b0-4136-8a34-7495ba00c352 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a1277} kind: Pod name: sequence-graph-a1277-6d9f5c6dd8-sqgk9 namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: 53213afd-bfbe-44e9-a0ce-5136a80871b2 kind: Event lastTimestamp: "2026-04-17T17:45:26Z" message: Started container sequence-graph-a1277 metadata: creationTimestamp: "2026-04-17T17:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:26Z" name: sequence-graph-a1277-6d9f5c6dd8-sqgk9.18a735fa397ce2c6 namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: 24a2b460-9188-4e99-9df5-312fc08ebf91 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a1277} kind: Pod name: sequence-graph-a1277-6d9f5c6dd8-sqgk9 namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: 53213afd-bfbe-44e9-a0ce-5136a80871b2 kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: Stopping container sequence-graph-a1277 metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:36Z" name: sequence-graph-a1277-6d9f5c6dd8-sqgk9.18a735fc728157ed namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: 4186fb27-75d6-485f-842d-a646dd207143 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:45:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a1277} kind: Pod name: sequence-graph-a1277-6d9f5c6dd8-sqgk9 namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: 53213afd-bfbe-44e9-a0ce-5136a80871b2 kind: Event lastTimestamp: "2026-04-17T17:46:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:46:03Z" name: sequence-graph-a1277-6d9f5c6dd8-sqgk9.18a735fcf0186f5f namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: bd812c9a-af1a-468b-957d-0ba1b6042ec2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a1277-6d9f5c6dd8 namespace: kserve-ci-e2e-test resourceVersion: "20479" uid: 59a15a7a-26a0-462b-bc69-8cbc6e1b32df kind: Event lastTimestamp: "2026-04-17T17:45:26Z" message: 'Created pod: sequence-graph-a1277-6d9f5c6dd8-sqgk9' metadata: creationTimestamp: "2026-04-17T17:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:26Z" name: sequence-graph-a1277-6d9f5c6dd8.18a735fa1b49de7b namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: ee377af3-984e-42b1-8bc7-e6e79a4ceec7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a1277 namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: 7634c0fe-7dbb-49b7-b6bc-4be9b1dee0db kind: Event lastTimestamp: "2026-04-17T17:45:26Z" message: Scaled up replica set sequence-graph-a1277-6d9f5c6dd8 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:26Z" name: sequence-graph-a1277.18a735fa1a794de2 namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: 9a467917-e269-4215-8151-61f9f2eb849b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a1277 namespace: kserve-ci-e2e-test resourceVersion: "20495" uid: da291a6e-7184-429d-8984-bb087e1d6326 kind: Event lastTimestamp: "2026-04-17T17:45:26Z" message: InferenceGraph [sequence-graph-a1277] is Ready metadata: creationTimestamp: "2026-04-17T17:45:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:26Z" name: sequence-graph-a1277.18a735fa1fd304b7 namespace: kserve-ci-e2e-test resourceVersion: "20498" uid: 4ff04376-3ce6-4a49-b426-8f3e97d6cbc6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:46:36.605764Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-dd512-7648f6b85c-9h6m9 namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: 99521847-f803-41bd-9c3b-2a7dcfe5411e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-dd512-7648f6b85c-9h6m9 to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:46:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:46:36Z" name: sequence-graph-dd512-7648f6b85c-9h6m9.18a7360a77a4315e namespace: kserve-ci-e2e-test resourceVersion: "21302" uid: 2fdbf300-fe4c-4da2-b246-62ffb8937d2e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:46:37Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-dd512-7648f6b85c-9h6m9 namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: 99521847-f803-41bd-9c3b-2a7dcfe5411e kind: Event lastTimestamp: "2026-04-17T17:46:37Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:46:37Z" name: sequence-graph-dd512-7648f6b85c-9h6m9.18a7360a90e518ad namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: 547233d5-74c6-490a-a35f-21aff7cd551d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dd512} kind: Pod name: sequence-graph-dd512-7648f6b85c-9h6m9 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 99521847-f803-41bd-9c3b-2a7dcfe5411e kind: Event lastTimestamp: "2026-04-17T17:46:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0589637b57a3315022b8ebf7fc1b0f9831e3a45e07a3fc2fab24137dd8f190f9" already present on machine metadata: creationTimestamp: "2026-04-17T17:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:46:37Z" name: sequence-graph-dd512-7648f6b85c-9h6m9.18a7360a92336dea namespace: kserve-ci-e2e-test resourceVersion: "21321" uid: a4fd2a3f-6f62-4375-a566-f71d7ffbc82c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dd512} kind: Pod name: sequence-graph-dd512-7648f6b85c-9h6m9 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 99521847-f803-41bd-9c3b-2a7dcfe5411e kind: Event lastTimestamp: "2026-04-17T17:46:37Z" message: 'Created container: sequence-graph-dd512' metadata: creationTimestamp: "2026-04-17T17:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:46:37Z" name: sequence-graph-dd512-7648f6b85c-9h6m9.18a7360a95838a41 namespace: kserve-ci-e2e-test resourceVersion: "21323" uid: 215bc70a-835c-4f7a-b6c3-6076d598b13c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dd512} kind: Pod name: sequence-graph-dd512-7648f6b85c-9h6m9 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 99521847-f803-41bd-9c3b-2a7dcfe5411e kind: Event lastTimestamp: "2026-04-17T17:46:37Z" message: Started container sequence-graph-dd512 metadata: creationTimestamp: "2026-04-17T17:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:46:37Z" name: sequence-graph-dd512-7648f6b85c-9h6m9.18a7360a95f58136 namespace: kserve-ci-e2e-test resourceVersion: "21324" uid: 85260d1f-f022-4cc1-af40-be446665a43f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dd512} kind: Pod name: sequence-graph-dd512-7648f6b85c-9h6m9 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 99521847-f803-41bd-9c3b-2a7dcfe5411e kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: Stopping container sequence-graph-dd512 metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:51Z" name: sequence-graph-dd512-7648f6b85c-9h6m9.18a7367d9f3d8728 namespace: kserve-ci-e2e-test resourceVersion: "24903" uid: c1bc869d-fa09-4814-8c00-2b117382c598 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:54:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-dd512} kind: Pod name: sequence-graph-dd512-7648f6b85c-9h6m9 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 99521847-f803-41bd-9c3b-2a7dcfe5411e kind: Event lastTimestamp: "2026-04-17T17:55:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:54:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:18Z" name: sequence-graph-dd512-7648f6b85c-9h6m9.18a7367e37bedbfc namespace: kserve-ci-e2e-test resourceVersion: "25430" uid: 884ce097-821a-475f-8e0e-0ea326950ec9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:46:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-dd512-7648f6b85c namespace: kserve-ci-e2e-test resourceVersion: "21293" uid: 9b0f924b-4946-4177-b8ad-c1842dde5f71 kind: Event lastTimestamp: "2026-04-17T17:46:36Z" message: 'Created pod: sequence-graph-dd512-7648f6b85c-9h6m9' metadata: creationTimestamp: "2026-04-17T17:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:36Z" name: sequence-graph-dd512-7648f6b85c.18a7360a771eb5d9 namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: 1e01575e-9c75-46cd-b93c-9aa6bf86c996 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:46:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-dd512 namespace: kserve-ci-e2e-test resourceVersion: "21290" uid: 5b666298-c080-4429-b159-4364a1fe5e69 kind: Event lastTimestamp: "2026-04-17T17:46:36Z" message: Scaled up replica set sequence-graph-dd512-7648f6b85c from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:36Z" name: sequence-graph-dd512.18a7360a763a0ab4 namespace: kserve-ci-e2e-test resourceVersion: "21295" uid: d0dc091d-282c-4b79-899e-b6f1812a8b9c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:46:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-dd512 namespace: kserve-ci-e2e-test resourceVersion: "21312" uid: d84497f2-4b80-4a40-8e56-0c6089480eaf kind: Event lastTimestamp: "2026-04-17T17:46:36Z" message: InferenceGraph [sequence-graph-dd512] is Ready metadata: creationTimestamp: "2026-04-17T17:46:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:46:36Z" name: sequence-graph-dd512.18a7360a7be8824f namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: 2e3fd2ed-c7a2-46be-af7c-b2353d4e3f52 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:46:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-dd512 namespace: kserve-ci-e2e-test resourceVersion: "21292" uid: 2e6e7ca4-8522-497d-9c4a-471beea6963f kind: Event lastTimestamp: "2026-04-17T17:46:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:51Z" name: sequence-graph-dd512.18a7360df4b16430 namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 90f7bd27-f8b1-4199-8583-ad5d2894d623 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:46:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-dd512 namespace: kserve-ci-e2e-test resourceVersion: "21292" uid: 2e6e7ca4-8522-497d-9c4a-471beea6963f kind: Event lastTimestamp: "2026-04-17T17:46:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:46:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:51Z" name: sequence-graph-dd512.18a7360df518ef69 namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: a5041e99-2722-4546-afd5-c974e30b4485 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:47:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-dd512 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 2e6e7ca4-8522-497d-9c4a-471beea6963f kind: Event lastTimestamp: "2026-04-17T17:47:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:47:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:47:06Z" name: sequence-graph-dd512.18a7361173c218f3 namespace: kserve-ci-e2e-test resourceVersion: "21539" uid: 5c2a289b-442f-4472-8354-fca38a417b37 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:47:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-dd512 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 2e6e7ca4-8522-497d-9c4a-471beea6963f kind: Event lastTimestamp: "2026-04-17T17:47:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:47:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:47:06Z" name: sequence-graph-dd512.18a73611742b564f namespace: kserve-ci-e2e-test resourceVersion: "21541" uid: a8e0fc17-4ccf-462c-9f29-b4dd0c902765 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:36:11.478809Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fa93d-58f6759648-qxwxn namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: 06cf717f-8ff8-48d5-bb9d-a6ce56b0bbc9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-fa93d-58f6759648-qxwxn to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:36:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:36:11Z" name: sequence-graph-fa93d-58f6759648-qxwxn.18a73578eb2bd54e namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: eae536be-e9d2-45d9-8b4e-bbe67aa195ee reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:11Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fa93d-58f6759648-qxwxn namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 06cf717f-8ff8-48d5-bb9d-a6ce56b0bbc9 kind: Event lastTimestamp: "2026-04-17T17:36:11Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:36:11Z" name: sequence-graph-fa93d-58f6759648-qxwxn.18a735790464bd7d namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 0da672a7-1807-4fec-a1f1-6d52edeb9ea9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fa93d} kind: Pod name: sequence-graph-fa93d-58f6759648-qxwxn namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: 06cf717f-8ff8-48d5-bb9d-a6ce56b0bbc9 kind: Event lastTimestamp: "2026-04-17T17:36:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0589637b57a3315022b8ebf7fc1b0f9831e3a45e07a3fc2fab24137dd8f190f9" already present on machine metadata: creationTimestamp: "2026-04-17T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:36:11Z" name: sequence-graph-fa93d-58f6759648-qxwxn.18a7357906e5a306 namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: fe1cd4c2-e99a-4621-81cb-6c947a62cb46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fa93d} kind: Pod name: sequence-graph-fa93d-58f6759648-qxwxn namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: 06cf717f-8ff8-48d5-bb9d-a6ce56b0bbc9 kind: Event lastTimestamp: "2026-04-17T17:36:11Z" message: 'Created container: sequence-graph-fa93d' metadata: creationTimestamp: "2026-04-17T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:36:11Z" name: sequence-graph-fa93d-58f6759648-qxwxn.18a735790a09fdac namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: 78cfb9ba-1e88-4711-a45a-11ff11b29b2e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fa93d} kind: Pod name: sequence-graph-fa93d-58f6759648-qxwxn namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: 06cf717f-8ff8-48d5-bb9d-a6ce56b0bbc9 kind: Event lastTimestamp: "2026-04-17T17:36:12Z" message: Started container sequence-graph-fa93d metadata: creationTimestamp: "2026-04-17T17:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:36:12Z" name: sequence-graph-fa93d-58f6759648-qxwxn.18a735790a78c5c1 namespace: kserve-ci-e2e-test resourceVersion: "16081" uid: fd4e14bd-72f0-4263-a2a8-ebd08f714455 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fa93d} kind: Pod name: sequence-graph-fa93d-58f6759648-qxwxn namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: 06cf717f-8ff8-48d5-bb9d-a6ce56b0bbc9 kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: Stopping container sequence-graph-fa93d metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:26Z" name: sequence-graph-fa93d-58f6759648-qxwxn.18a735ec179b6cf8 namespace: kserve-ci-e2e-test resourceVersion: "19667" uid: 9ac87b76-0dab-4425-8611-c4cdaadf9d9d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:44:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fa93d} kind: Pod name: sequence-graph-fa93d-58f6759648-qxwxn namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: 06cf717f-8ff8-48d5-bb9d-a6ce56b0bbc9 kind: Event lastTimestamp: "2026-04-17T17:44:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:53Z" name: sequence-graph-fa93d-58f6759648-qxwxn.18a735ecaa93d487 namespace: kserve-ci-e2e-test resourceVersion: "20189" uid: 09e42959-28d6-4cd1-ab6e-c02a851b1ef7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-fa93d-58f6759648 namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 1d28f07e-fcf7-4e4c-8175-1777263d3187 kind: Event lastTimestamp: "2026-04-17T17:36:11Z" message: 'Created pod: sequence-graph-fa93d-58f6759648-qxwxn' metadata: creationTimestamp: "2026-04-17T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:36:11Z" name: sequence-graph-fa93d-58f6759648.18a73578ead1f5ff namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: f23b8d21-b9d5-44ce-829d-973bb16d5298 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-fa93d namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 3ae5585e-5dff-4613-998d-9a51641951bf kind: Event lastTimestamp: "2026-04-17T17:36:11Z" message: Scaled up replica set sequence-graph-fa93d-58f6759648 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:36:11Z" name: sequence-graph-fa93d.18a73578ea14767c namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: b035baa3-5e52-497c-a5e9-e6a2432fc5ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-fa93d namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 9b22e0db-e938-4557-83f5-2c2df79b5ec3 kind: Event lastTimestamp: "2026-04-17T17:36:11Z" message: InferenceGraph [sequence-graph-fa93d] is Ready metadata: creationTimestamp: "2026-04-17T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:36:11Z" name: sequence-graph-fa93d.18a73578ef520339 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 9c27d4fb-a677-4ecc-bc0d-b6ec23af5f98 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-fa93d namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: 9b22e0db-e938-4557-83f5-2c2df79b5ec3 kind: Event lastTimestamp: "2026-04-17T17:36:11Z" message: 'Failed to update status for InferenceGraph "sequence-graph-fa93d": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-fa93d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:36:11Z" name: sequence-graph-fa93d.18a73578f16dd2a6 namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: e9bda600-f325-4e11-bc4c-0bda93a53dc1 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-fa93d namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: 9b22e0db-e938-4557-83f5-2c2df79b5ec3 kind: Event lastTimestamp: "2026-04-17T17:36:11Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-fa93d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:36:11Z" name: sequence-graph-fa93d.18a73578f16e54cd namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 36141b0c-e1a3-457e-bd23-44717e9e311d reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fa93d namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 35d54f20-a18c-4909-8911-9f1e6ca07ee9 kind: Event lastTimestamp: "2026-04-17T17:36:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:36:26Z" name: sequence-graph-fa93d.18a7357c688727e7 namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: bf7f080b-3b45-4dd2-abcc-821469edff0d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fa93d namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 35d54f20-a18c-4909-8911-9f1e6ca07ee9 kind: Event lastTimestamp: "2026-04-17T17:36:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:36:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:36:26Z" name: sequence-graph-fa93d.18a7357c68d449ba namespace: kserve-ci-e2e-test resourceVersion: "16188" uid: 6ebf056b-c9db-4a67-adab-d88ce7b58884 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fa93d namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: 35d54f20-a18c-4909-8911-9f1e6ca07ee9 kind: Event lastTimestamp: "2026-04-17T17:36:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:36:41Z" name: sequence-graph-fa93d.18a7357fe780e640 namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 5e1c22cb-a259-4b3d-a41d-66d89adf433b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-fa93d namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: 35d54f20-a18c-4909-8911-9f1e6ca07ee9 kind: Event lastTimestamp: "2026-04-17T17:36:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:36:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:36:41Z" name: sequence-graph-fa93d.18a7357fe7ef4609 namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 27fb47ce-805c-412e-a3ef-858478487e4a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:56:12.266247Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a935a-584b858c64-b458q namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: 7d71cc4b-fcea-41f7-b193-ab0017ab8850 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-a935a-584b858c64-b458q to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:56:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:56:12Z" name: splitter-graph-a935a-584b858c64-b458q.18a736907fae0bd7 namespace: kserve-ci-e2e-test resourceVersion: "25948" uid: 02664fee-02da-40b9-b234-7f3edb9cd63f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:56:12Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a935a-584b858c64-b458q namespace: kserve-ci-e2e-test resourceVersion: "25944" uid: 7d71cc4b-fcea-41f7-b193-ab0017ab8850 kind: Event lastTimestamp: "2026-04-17T17:56:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-a935a-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:56:12Z" name: splitter-graph-a935a-584b858c64-b458q.18a736908834d406 namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: f08399bf-8d52-4710-a302-bc2346cd3a5b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:56:13Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a935a-584b858c64-b458q namespace: kserve-ci-e2e-test resourceVersion: "25952" uid: 7d71cc4b-fcea-41f7-b193-ab0017ab8850 kind: Event lastTimestamp: "2026-04-17T17:56:13Z" message: Add eth0 [10.132.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:56:13Z" name: splitter-graph-a935a-584b858c64-b458q.18a73690bcaf955a namespace: kserve-ci-e2e-test resourceVersion: "25968" uid: 5b441483-4fcb-4ae1-a8e1-9d534783383c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a935a} kind: Pod name: splitter-graph-a935a-584b858c64-b458q namespace: kserve-ci-e2e-test resourceVersion: "25944" uid: 7d71cc4b-fcea-41f7-b193-ab0017ab8850 kind: Event lastTimestamp: "2026-04-17T17:56:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0589637b57a3315022b8ebf7fc1b0f9831e3a45e07a3fc2fab24137dd8f190f9" already present on machine metadata: creationTimestamp: "2026-04-17T17:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:56:13Z" name: splitter-graph-a935a-584b858c64-b458q.18a73690be70b60a namespace: kserve-ci-e2e-test resourceVersion: "25971" uid: 293a2c39-2aae-478c-87a8-2180cda859e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a935a} kind: Pod name: splitter-graph-a935a-584b858c64-b458q namespace: kserve-ci-e2e-test resourceVersion: "25944" uid: 7d71cc4b-fcea-41f7-b193-ab0017ab8850 kind: Event lastTimestamp: "2026-04-17T17:56:13Z" message: 'Created container: splitter-graph-a935a' metadata: creationTimestamp: "2026-04-17T17:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:56:13Z" name: splitter-graph-a935a-584b858c64-b458q.18a73690c1e286e3 namespace: kserve-ci-e2e-test resourceVersion: "25972" uid: ad7af9ec-b665-4756-84dc-aeb99a365d8e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a935a} kind: Pod name: splitter-graph-a935a-584b858c64-b458q namespace: kserve-ci-e2e-test resourceVersion: "25944" uid: 7d71cc4b-fcea-41f7-b193-ab0017ab8850 kind: Event lastTimestamp: "2026-04-17T17:56:13Z" message: Started container splitter-graph-a935a metadata: creationTimestamp: "2026-04-17T17:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:56:13Z" name: splitter-graph-a935a-584b858c64-b458q.18a73690c251c844 namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 6a99f956-0630-486e-b83c-fe4ed0957fca reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T18:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a935a} kind: Pod name: splitter-graph-a935a-584b858c64-b458q namespace: kserve-ci-e2e-test resourceVersion: "25944" uid: 7d71cc4b-fcea-41f7-b193-ab0017ab8850 kind: Event lastTimestamp: "2026-04-17T18:04:26Z" message: Stopping container splitter-graph-a935a metadata: creationTimestamp: "2026-04-17T18:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:04:26Z" name: splitter-graph-a935a-584b858c64-b458q.18a737039f1a8502 namespace: kserve-ci-e2e-test resourceVersion: "29695" uid: 1dedf18e-5c0e-4b98-9df6-1f79f14ea3af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T18:04:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a935a} kind: Pod name: splitter-graph-a935a-584b858c64-b458q namespace: kserve-ci-e2e-test resourceVersion: "25944" uid: 7d71cc4b-fcea-41f7-b193-ab0017ab8850 kind: Event lastTimestamp: "2026-04-17T18:04:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T18:04:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:04:54Z" name: splitter-graph-a935a-584b858c64-b458q.18a737044f9d9172 namespace: kserve-ci-e2e-test resourceVersion: "29929" uid: b51a35d9-6a6c-4535-be50-23c18112c33f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:56:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-a935a-584b858c64 namespace: kserve-ci-e2e-test resourceVersion: "25939" uid: 206b1ebc-ff54-40c6-8353-27c6e7e07024 kind: Event lastTimestamp: "2026-04-17T17:56:12Z" message: 'Created pod: splitter-graph-a935a-584b858c64-b458q' metadata: creationTimestamp: "2026-04-17T17:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:12Z" name: splitter-graph-a935a-584b858c64.18a736907eab0e60 namespace: kserve-ci-e2e-test resourceVersion: "25945" uid: 3e6fe75b-4892-4750-93e8-451b80340d3e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:56:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-a935a namespace: kserve-ci-e2e-test resourceVersion: "25937" uid: cf203938-805b-457d-8259-adeef70be509 kind: Event lastTimestamp: "2026-04-17T17:56:12Z" message: Scaled up replica set splitter-graph-a935a-584b858c64 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:12Z" name: splitter-graph-a935a.18a736907d62be4e namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 4b621a9f-786e-48c1-bc37-2c650c123c1e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:56:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a935a namespace: kserve-ci-e2e-test resourceVersion: "25960" uid: 49ccd46e-4f08-4f95-bb44-58a3d762b208 kind: Event lastTimestamp: "2026-04-17T17:56:12Z" message: InferenceGraph [splitter-graph-a935a] is Ready metadata: creationTimestamp: "2026-04-17T17:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:56:12Z" name: splitter-graph-a935a.18a736908b6424fe namespace: kserve-ci-e2e-test resourceVersion: "25962" uid: d2509a77-3338-477d-bc45-b5387ebbbf14 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:56:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-a935a namespace: kserve-ci-e2e-test resourceVersion: "25938" uid: 69b35e3b-d8dd-4ce7-857e-41320a8985de kind: Event lastTimestamp: "2026-04-17T17:56:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:27Z" name: splitter-graph-a935a.18a73693fc0e3bcc namespace: kserve-ci-e2e-test resourceVersion: "26241" uid: aa3a6e4d-da76-432c-becd-c0cc4554b980 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:56:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-a935a namespace: kserve-ci-e2e-test resourceVersion: "25938" uid: 69b35e3b-d8dd-4ce7-857e-41320a8985de kind: Event lastTimestamp: "2026-04-17T17:56:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:56:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:27Z" name: splitter-graph-a935a.18a73693fc881770 namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 865dfe92-6f62-4591-8452-c649f69b3352 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:56:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-a935a namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: 69b35e3b-d8dd-4ce7-857e-41320a8985de kind: Event lastTimestamp: "2026-04-17T17:56:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:42Z" name: splitter-graph-a935a.18a736977b910aae namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: d6bc8290-e12f-4ff8-be64-c656cc4a63c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:56:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-a935a namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: 69b35e3b-d8dd-4ce7-857e-41320a8985de kind: Event lastTimestamp: "2026-04-17T17:56:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:56:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:42Z" name: splitter-graph-a935a.18a736977c07fa7b namespace: kserve-ci-e2e-test resourceVersion: "26480" uid: c47fab30-f951-450b-8021-f64f95cc97dd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:55:01.870726Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e4a99-7d99d74f5f-459vd namespace: kserve-ci-e2e-test resourceVersion: "25119" uid: fc5a3cf8-71bb-43ef-8350-472babdebd07 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-e4a99-7d99d74f5f-459vd to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:55:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:55:01Z" name: splitter-graph-e4a99-7d99d74f5f-459vd.18a736801bc7a7fe namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: 7c1de26b-6b2c-4cf7-b91f-2a7cc0e820dc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:02Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e4a99-7d99d74f5f-459vd namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: fc5a3cf8-71bb-43ef-8350-472babdebd07 kind: Event lastTimestamp: "2026-04-17T17:55:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-e4a99-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:02Z" name: splitter-graph-e4a99-7d99d74f5f-459vd.18a736802b7efd94 namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: e813efa3-7b76-4151-8229-f30a20a674e8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:02Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e4a99-7d99d74f5f-459vd namespace: kserve-ci-e2e-test resourceVersion: "25128" uid: fc5a3cf8-71bb-43ef-8350-472babdebd07 kind: Event lastTimestamp: "2026-04-17T17:55:02Z" message: Add eth0 [10.132.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:55:02Z" name: splitter-graph-e4a99-7d99d74f5f-459vd.18a7368058f7990b namespace: kserve-ci-e2e-test resourceVersion: "25150" uid: c5549297-cc3f-4764-bb8b-888a36bb867c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e4a99} kind: Pod name: splitter-graph-e4a99-7d99d74f5f-459vd namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: fc5a3cf8-71bb-43ef-8350-472babdebd07 kind: Event lastTimestamp: "2026-04-17T17:55:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0589637b57a3315022b8ebf7fc1b0f9831e3a45e07a3fc2fab24137dd8f190f9" already present on machine metadata: creationTimestamp: "2026-04-17T17:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:02Z" name: splitter-graph-e4a99-7d99d74f5f-459vd.18a736805a640160 namespace: kserve-ci-e2e-test resourceVersion: "25153" uid: 97977c12-faa0-47c5-87c6-cd34a0975ea3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e4a99} kind: Pod name: splitter-graph-e4a99-7d99d74f5f-459vd namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: fc5a3cf8-71bb-43ef-8350-472babdebd07 kind: Event lastTimestamp: "2026-04-17T17:55:02Z" message: 'Created container: splitter-graph-e4a99' metadata: creationTimestamp: "2026-04-17T17:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:02Z" name: splitter-graph-e4a99-7d99d74f5f-459vd.18a736805e0def1c namespace: kserve-ci-e2e-test resourceVersion: "25154" uid: eaf2ce8a-0088-4a5e-8718-00a876859e3b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e4a99} kind: Pod name: splitter-graph-e4a99-7d99d74f5f-459vd namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: fc5a3cf8-71bb-43ef-8350-472babdebd07 kind: Event lastTimestamp: "2026-04-17T17:55:02Z" message: Started container splitter-graph-e4a99 metadata: creationTimestamp: "2026-04-17T17:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:02Z" name: splitter-graph-e4a99-7d99d74f5f-459vd.18a736805e7f42a5 namespace: kserve-ci-e2e-test resourceVersion: "25155" uid: 23352af6-a3f5-4b91-8171-5885ee2f684f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e4a99} kind: Pod name: splitter-graph-e4a99-7d99d74f5f-459vd namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: fc5a3cf8-71bb-43ef-8350-472babdebd07 kind: Event lastTimestamp: "2026-04-17T17:55:11Z" message: Stopping container splitter-graph-e4a99 metadata: creationTimestamp: "2026-04-17T17:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:11Z" name: splitter-graph-e4a99-7d99d74f5f-459vd.18a736827491879a namespace: kserve-ci-e2e-test resourceVersion: "25251" uid: 03b58052-7165-41ec-9c46-f98327950738 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e4a99} kind: Pod name: splitter-graph-e4a99-7d99d74f5f-459vd namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: fc5a3cf8-71bb-43ef-8350-472babdebd07 kind: Event lastTimestamp: "2026-04-17T17:55:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:39Z" name: splitter-graph-e4a99-7d99d74f5f-459vd.18a7368308b9f2cb namespace: kserve-ci-e2e-test resourceVersion: "25605" uid: 1349dc10-64b8-4ec7-9e41-c3cbd9788857 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-e4a99-7d99d74f5f namespace: kserve-ci-e2e-test resourceVersion: "25114" uid: 7152fc6a-e707-4453-a146-eaad971f4d02 kind: Event lastTimestamp: "2026-04-17T17:55:01Z" message: 'Created pod: splitter-graph-e4a99-7d99d74f5f-459vd' metadata: creationTimestamp: "2026-04-17T17:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:01Z" name: splitter-graph-e4a99-7d99d74f5f.18a736801b46954a namespace: kserve-ci-e2e-test resourceVersion: "25120" uid: 43d92e68-161a-4283-a165-04d4ced71e8d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-e4a99 namespace: kserve-ci-e2e-test resourceVersion: "25112" uid: b278d1b2-a5e8-4acf-a1ed-6f0ed8fc6d83 kind: Event lastTimestamp: "2026-04-17T17:55:01Z" message: Scaled up replica set splitter-graph-e4a99-7d99d74f5f from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:01Z" name: splitter-graph-e4a99.18a736801aa21592 namespace: kserve-ci-e2e-test resourceVersion: "25116" uid: 76e207fd-1bfd-4a71-90a4-f39050f8bc4a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-e4a99 namespace: kserve-ci-e2e-test resourceVersion: "25130" uid: 03c05cf4-cf0c-46c6-8246-e80f7ef4fa50 kind: Event lastTimestamp: "2026-04-17T17:55:01Z" message: InferenceGraph [splitter-graph-e4a99] is Ready metadata: creationTimestamp: "2026-04-17T17:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:55:01Z" name: splitter-graph-e4a99.18a736801e3078a6 namespace: kserve-ci-e2e-test resourceVersion: "25132" uid: 65da37cd-8eb4-46cc-9584-f9e7e475b6d2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:54:51.361344Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0283f-predictor-bfc486968-gvj27 namespace: kserve-ci-e2e-test resourceVersion: "24943" uid: d97c20bd-af97-487f-8815-e0497431b94d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0283f-predictor-bfc486968-gvj27 to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:54:51Z" name: success-200-isvc-0283f-predictor-bfc486968-gvj27.18a7367da95f3eac namespace: kserve-ci-e2e-test resourceVersion: "24950" uid: 5b471d41-d830-4c4c-8738-429218c48a89 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0283f-predictor-bfc486968-gvj27 namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: d97c20bd-af97-487f-8815-e0497431b94d kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-0283f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:51Z" name: success-200-isvc-0283f-predictor-bfc486968-gvj27.18a7367dba929e86 namespace: kserve-ci-e2e-test resourceVersion: "24993" uid: b42ce5e9-68b4-4e87-9acd-8073803cc600 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0283f-predictor-bfc486968-gvj27 namespace: kserve-ci-e2e-test resourceVersion: "24960" uid: d97c20bd-af97-487f-8815-e0497431b94d kind: Event lastTimestamp: "2026-04-17T17:54:52Z" message: Add eth0 [10.132.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:54:52Z" name: success-200-isvc-0283f-predictor-bfc486968-gvj27.18a7367de6be0917 namespace: kserve-ci-e2e-test resourceVersion: "25015" uid: 6a78b5f1-dd9d-45ef-8615-d76cae07b2af reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0283f-predictor-bfc486968-gvj27 namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: d97c20bd-af97-487f-8815-e0497431b94d kind: Event lastTimestamp: "2026-04-17T17:54:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:52Z" name: success-200-isvc-0283f-predictor-bfc486968-gvj27.18a7367de811e298 namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: 821487e9-519c-4fa2-8bb2-5b9ee215724b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0283f-predictor-bfc486968-gvj27 namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: d97c20bd-af97-487f-8815-e0497431b94d kind: Event lastTimestamp: "2026-04-17T17:54:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:52Z" name: success-200-isvc-0283f-predictor-bfc486968-gvj27.18a7367deb2f37ea namespace: kserve-ci-e2e-test resourceVersion: "25018" uid: 9fa72f60-c584-42fb-be31-df6f9523df4b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0283f-predictor-bfc486968-gvj27 namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: d97c20bd-af97-487f-8815-e0497431b94d kind: Event lastTimestamp: "2026-04-17T17:54:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:52Z" name: success-200-isvc-0283f-predictor-bfc486968-gvj27.18a7367debb1a39e namespace: kserve-ci-e2e-test resourceVersion: "25019" uid: 1e8ee62f-a4d8-426b-835c-b78696274dbf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0283f-predictor-bfc486968-gvj27 namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: d97c20bd-af97-487f-8815-e0497431b94d kind: Event lastTimestamp: "2026-04-17T17:54:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:52Z" name: success-200-isvc-0283f-predictor-bfc486968-gvj27.18a7367debbd5247 namespace: kserve-ci-e2e-test resourceVersion: "25020" uid: a6c0dafc-f248-4c42-9fdb-29b83ad36429 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0283f-predictor-bfc486968-gvj27 namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: d97c20bd-af97-487f-8815-e0497431b94d kind: Event lastTimestamp: "2026-04-17T17:54:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:52Z" name: success-200-isvc-0283f-predictor-bfc486968-gvj27.18a7367df068d32c namespace: kserve-ci-e2e-test resourceVersion: "25023" uid: b4105d2c-93b5-4b45-af8a-ab724b34f6d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0283f-predictor-bfc486968-gvj27 namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: d97c20bd-af97-487f-8815-e0497431b94d kind: Event lastTimestamp: "2026-04-17T17:54:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:54:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:52Z" name: success-200-isvc-0283f-predictor-bfc486968-gvj27.18a7367df0dd9075 namespace: kserve-ci-e2e-test resourceVersion: "25024" uid: 5dd5048e-cb69-4877-ada3-b640203d2e3a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:54:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0283f-predictor-bfc486968-gvj27 namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: d97c20bd-af97-487f-8815-e0497431b94d kind: Event lastTimestamp: "2026-04-17T17:55:30Z" message: 'Readiness probe failed: dial tcp 10.132.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:54:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:30Z" name: success-200-isvc-0283f-predictor-bfc486968-gvj27.18a7367e5e50bbc1 namespace: kserve-ci-e2e-test resourceVersion: "25529" uid: 7695272a-b6ce-4588-b85c-8dd262b9f3f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T18:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0283f-predictor-bfc486968-gvj27 namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: d97c20bd-af97-487f-8815-e0497431b94d kind: Event lastTimestamp: "2026-04-17T18:12:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T18:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:12:10Z" name: success-200-isvc-0283f-predictor-bfc486968-gvj27.18a7376fb4002d68 namespace: kserve-ci-e2e-test resourceVersion: "32926" uid: 0c3bcf64-a7c0-4325-be4a-11a4c7c2433c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T18:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0283f-predictor-bfc486968-gvj27 namespace: kserve-ci-e2e-test resourceVersion: "24947" uid: d97c20bd-af97-487f-8815-e0497431b94d kind: Event lastTimestamp: "2026-04-17T18:12:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T18:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:12:10Z" name: success-200-isvc-0283f-predictor-bfc486968-gvj27.18a7376fb4009eda namespace: kserve-ci-e2e-test resourceVersion: "32927" uid: 5d7a3207-e30e-44c2-8582-d49175e91394 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0283f-predictor-bfc486968 namespace: kserve-ci-e2e-test resourceVersion: "24931" uid: 5daf7190-1e0c-4e2c-85ee-e69674b75cd3 kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: 'Created pod: success-200-isvc-0283f-predictor-bfc486968-gvj27' metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:54:51Z" name: success-200-isvc-0283f-predictor-bfc486968.18a7367da8e99217 namespace: kserve-ci-e2e-test resourceVersion: "24946" uid: 4f88159d-0f4e-471d-8095-20b1224e19c7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0283f-predictor namespace: kserve-ci-e2e-test resourceVersion: "24927" uid: 23789138-99ba-46fd-8dbd-89cde97891f8 kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: Scaled up replica set success-200-isvc-0283f-predictor-bfc486968 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:54:51Z" name: success-200-isvc-0283f-predictor.18a7367da6f52ee6 namespace: kserve-ci-e2e-test resourceVersion: "24933" uid: 90475490-275e-404d-9a65-0dfda9b54751 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:55:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0283f-predictor namespace: kserve-ci-e2e-test resourceVersion: "24929" uid: ba65231e-0267-4624-be57-8e7f50a7ef8a kind: Event lastTimestamp: "2026-04-17T17:55:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:21Z" name: success-200-isvc-0283f-predictor.18a73681258114ef namespace: kserve-ci-e2e-test resourceVersion: "25454" uid: 09875098-f03b-4b37-a307-a9d2d33058fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:55:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0283f-predictor namespace: kserve-ci-e2e-test resourceVersion: "24929" uid: ba65231e-0267-4624-be57-8e7f50a7ef8a kind: Event lastTimestamp: "2026-04-17T17:55:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:21Z" name: success-200-isvc-0283f-predictor.18a7368125edc51b namespace: kserve-ci-e2e-test resourceVersion: "25455" uid: 6c870afa-568c-4100-a962-71dc99a7dbd5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:55:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0283f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25181" uid: ba65231e-0267-4624-be57-8e7f50a7ef8a kind: Event lastTimestamp: "2026-04-17T17:56:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:06Z" name: success-200-isvc-0283f-predictor.18a73688242403ec namespace: kserve-ci-e2e-test resourceVersion: "25890" uid: c9a2eab2-4340-4009-b23c-a81c9339b676 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:55:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0283f-predictor namespace: kserve-ci-e2e-test resourceVersion: "25181" uid: ba65231e-0267-4624-be57-8e7f50a7ef8a kind: Event lastTimestamp: "2026-04-17T17:56:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:06Z" name: success-200-isvc-0283f-predictor.18a7368824906c37 namespace: kserve-ci-e2e-test resourceVersion: "25891" uid: 5fa32ddd-ffcb-40d7-8370-895e5dfdf0fd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0283f namespace: kserve-ci-e2e-test resourceVersion: "24905" uid: f89b4717-2468-4fce-b4d1-81c9661d8416 kind: Event lastTimestamp: "2026-04-17T17:55:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:55:40Z" name: success-200-isvc-0283f.18a7367da6cd69e1 namespace: kserve-ci-e2e-test resourceVersion: "25631" uid: 1cb6bbab-9a5a-4933-a6d0-6799d961b233 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0283f namespace: kserve-ci-e2e-test resourceVersion: "24905" uid: f89b4717-2468-4fce-b4d1-81c9661d8416 kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-0283f": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0283f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:54:51Z" name: success-200-isvc-0283f.18a7367da71de43d namespace: kserve-ci-e2e-test resourceVersion: "24935" uid: eead3090-8143-4ca5-af3e-1d0584d1f6ec reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0283f namespace: kserve-ci-e2e-test resourceVersion: "24905" uid: f89b4717-2468-4fce-b4d1-81c9661d8416 kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0283f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:54:51Z" name: success-200-isvc-0283f.18a7367da71e3700 namespace: kserve-ci-e2e-test resourceVersion: "24936" uid: 746a01c0-2de4-44bd-a649-463be4da4ecc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0283f namespace: kserve-ci-e2e-test resourceVersion: "25626" uid: f89b4717-2468-4fce-b4d1-81c9661d8416 kind: Event lastTimestamp: "2026-04-17T17:55:40Z" message: InferenceService [success-200-isvc-0283f] is Ready metadata: creationTimestamp: "2026-04-17T17:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:55:40Z" name: success-200-isvc-0283f.18a736891997271e namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: 826e7c58-e75f-4cb6-8789-7e2377f4d7ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:32:51.459304Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13720" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-235f3-predictor-5d88b4c684-j25qw to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:32:51Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7354a591479a0 namespace: kserve-ci-e2e-test resourceVersion: "13724" uid: 0ee70a2f-e532-4a37-a40b-a1db776eb940 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13731" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:32:51Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7354a7244a731 namespace: kserve-ci-e2e-test resourceVersion: "13784" uid: 90a9248c-afa3-4e51-9547-8b37e83e88bc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:32:51Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7354a73acbdbc namespace: kserve-ci-e2e-test resourceVersion: "13786" uid: 49543ecd-790d-43ae-b429-27a3341f726b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: "2026-04-17T17:33:03Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" in 11.188s (11.188s including waiting). Image size: 1333697635 bytes.' metadata: creationTimestamp: "2026-04-17T17:33:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:03Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7354d0e953d58 namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: 8d22c024-8793-4f9c-ae68-9cea634796eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: "2026-04-17T17:33:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:33:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:03Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7354d11cde37e namespace: kserve-ci-e2e-test resourceVersion: "13950" uid: 41544f62-56a8-4bee-93d9-0afcb5f9b01c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: "2026-04-17T17:33:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:33:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:03Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7354d12409c73 namespace: kserve-ci-e2e-test resourceVersion: "13951" uid: c34026d2-7446-4b95-9424-efd6388d62b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: "2026-04-17T17:33:03Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-17T17:33:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:03Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7354d12551ddd namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: a8aabd76-ddfd-4b28-9856-de2575e98dfb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: "2026-04-17T17:33:06Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.925s (2.925s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-17T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:06Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7354dc0b7327e namespace: kserve-ci-e2e-test resourceVersion: "13992" uid: 46a10322-0afe-4ce0-bd54-c7f4792bb793 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: "2026-04-17T17:33:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:06Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7354dc402c049 namespace: kserve-ci-e2e-test resourceVersion: "13994" uid: d03aef0e-a548-4175-9a98-4595abf07b8f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: "2026-04-17T17:33:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:33:06Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7354dc48d180d namespace: kserve-ci-e2e-test resourceVersion: "13995" uid: c10fa959-b5d7-4612-801e-f40ffe278b63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T17:33:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7354df4e2b676 namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: ca1f748b-d8a2-4393-8742-ca68bd55c8e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:21Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7355f541c50fd namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: 641b8b22-0d6e-478d-a4f1-73745f196a9c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:21Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7355f54201846 namespace: kserve-ci-e2e-test resourceVersion: "14760" uid: 4cf525bc-f491-4de4-81d8-ab7cc0076f3f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw namespace: kserve-ci-e2e-test resourceVersion: "13722" uid: 1759c02c-89ff-4873-a196-cd59e3cc1f1c kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: success-200-isvc-235f3-predictor-5d88b4c684-j25qw.18a7355fa6fe6321 namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 206ed6fc-2dad-4677-b9bf-fbeb0e56cea7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-235f3-predictor-5d88b4c684 namespace: kserve-ci-e2e-test resourceVersion: "13710" uid: 28407d45-f95b-4cad-9b26-2ed2a6ff8efc kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: 'Created pod: success-200-isvc-235f3-predictor-5d88b4c684-j25qw' metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:32:51Z" name: success-200-isvc-235f3-predictor-5d88b4c684.18a7354a58914228 namespace: kserve-ci-e2e-test resourceVersion: "13721" uid: 8394b041-1e8f-4c3f-bdfa-f40ac673184a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-235f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "13708" uid: 943ea834-64a8-4ba4-ba74-61ba8fc0d89e kind: Event lastTimestamp: "2026-04-17T17:32:51Z" message: Scaled up replica set success-200-isvc-235f3-predictor-5d88b4c684 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:32:51Z" name: success-200-isvc-235f3-predictor.18a7354a536efbd8 namespace: kserve-ci-e2e-test resourceVersion: "13712" uid: 0f7d8b5f-5f3b-4a83-b0df-58b2354c202f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-235f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "13711" uid: 6b369baa-1695-4bb9-886b-72b1203c5eee kind: Event lastTimestamp: "2026-04-17T17:33:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:33:36Z" name: success-200-isvc-235f3-predictor.18a7354dd2613673 namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: 339e4182-0c03-4986-8ae3-07fa13d01253 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-235f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "13711" uid: 6b369baa-1695-4bb9-886b-72b1203c5eee kind: Event lastTimestamp: "2026-04-17T17:33:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:33:36Z" name: success-200-isvc-235f3-predictor.18a7354dd2cb25ea namespace: kserve-ci-e2e-test resourceVersion: "14299" uid: ffb4c1b3-2ed3-4446-be6b-8a56c1064618 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-235f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14002" uid: 6b369baa-1695-4bb9-886b-72b1203c5eee kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:21Z" name: success-200-isvc-235f3-predictor.18a735584f225b1a namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: cc733cc0-daf5-4ac0-8a26-8ae2f995cde3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:33:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-235f3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14002" uid: 6b369baa-1695-4bb9-886b-72b1203c5eee kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:21Z" name: success-200-isvc-235f3-predictor.18a735584f99fcd7 namespace: kserve-ci-e2e-test resourceVersion: "14733" uid: db1df484-e4df-4d41-b403-a36f5becc694 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:32:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-235f3 namespace: kserve-ci-e2e-test resourceVersion: "13694" uid: ecf27c4e-3c0f-4875-8b67-33dd6c03211b kind: Event lastTimestamp: "2026-04-17T17:33:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:33:53Z" name: success-200-isvc-235f3.18a7354a539ecfb2 namespace: kserve-ci-e2e-test resourceVersion: "14440" uid: 1552a086-9c8a-4982-9d7c-fe570e2d2f5d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:33:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-235f3 namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: ecf27c4e-3c0f-4875-8b67-33dd6c03211b kind: Event lastTimestamp: "2026-04-17T17:33:53Z" message: InferenceService [success-200-isvc-235f3] is Ready metadata: creationTimestamp: "2026-04-17T17:33:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:33:53Z" name: success-200-isvc-235f3.18a73558af8ddec9 namespace: kserve-ci-e2e-test resourceVersion: "14439" uid: 0ebc4195-fa79-474f-b44e-420aa680852f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:44:46.942124Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65 namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: 33b7e886-f7fd-4627-9183-0751e15881f5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-313c7-predictor-5bddb7798d-j8c65 to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:44:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:44:46Z" name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65.18a735f0ef2dfce1 namespace: kserve-ci-e2e-test resourceVersion: "20051" uid: 5c38b302-eef5-43c7-9533-ddb4fb85f43b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65 namespace: kserve-ci-e2e-test resourceVersion: "20058" uid: 33b7e886-f7fd-4627-9183-0751e15881f5 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:44:47Z" name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65.18a735f108be1bda namespace: kserve-ci-e2e-test resourceVersion: "20105" uid: a25380b9-2af6-4c82-880a-9aa0b79a8cc4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 33b7e886-f7fd-4627-9183-0751e15881f5 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65.18a735f10a2d80b1 namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: 78502554-6cc3-4631-b342-3089dc12901b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 33b7e886-f7fd-4627-9183-0751e15881f5 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65.18a735f10db34966 namespace: kserve-ci-e2e-test resourceVersion: "20109" uid: d16610be-0d4e-4e0c-99f6-ee5e5d531e84 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 33b7e886-f7fd-4627-9183-0751e15881f5 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65.18a735f10e240e61 namespace: kserve-ci-e2e-test resourceVersion: "20110" uid: 78d5dc20-5cd2-48f5-898b-495cffdb218c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 33b7e886-f7fd-4627-9183-0751e15881f5 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65.18a735f10e2fbf81 namespace: kserve-ci-e2e-test resourceVersion: "20112" uid: 615cb8cb-b8fb-45ec-85db-d6122003f91e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 33b7e886-f7fd-4627-9183-0751e15881f5 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65.18a735f110b6e2ba namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: 0a8ad4d0-33f3-4f27-97d3-c3fc75bed2f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 33b7e886-f7fd-4627-9183-0751e15881f5 kind: Event lastTimestamp: "2026-04-17T17:44:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:44:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:47Z" name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65.18a735f1112563d9 namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: 596958fe-2802-4cab-bbdf-d6d272aac59a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:44:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 33b7e886-f7fd-4627-9183-0751e15881f5 kind: Event lastTimestamp: "2026-04-17T17:45:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:25Z" name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65.18a735f17f70e775 namespace: kserve-ci-e2e-test resourceVersion: "20459" uid: b6f468a7-4d5e-4162-a094-6139bac3083d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 33b7e886-f7fd-4627-9183-0751e15881f5 kind: Event lastTimestamp: "2026-04-17T17:54:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:01Z" name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65.18a736721d38fbba namespace: kserve-ci-e2e-test resourceVersion: "24391" uid: 21765291-eeae-40bb-8b2a-4b15fd23df06 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65 namespace: kserve-ci-e2e-test resourceVersion: "20049" uid: 33b7e886-f7fd-4627-9183-0751e15881f5 kind: Event lastTimestamp: "2026-04-17T17:54:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:01Z" name: success-200-isvc-313c7-predictor-5bddb7798d-j8c65.18a736721d39c958 namespace: kserve-ci-e2e-test resourceVersion: "24393" uid: 0bfcbbab-a5a0-4247-9415-7a9876daba5c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-313c7-predictor-5bddb7798d namespace: kserve-ci-e2e-test resourceVersion: "20031" uid: fece0e0e-81d2-4991-aaed-83dfc320269f kind: Event lastTimestamp: "2026-04-17T17:44:46Z" message: 'Created pod: success-200-isvc-313c7-predictor-5bddb7798d-j8c65' metadata: creationTimestamp: "2026-04-17T17:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:46Z" name: success-200-isvc-313c7-predictor-5bddb7798d.18a735f0ee9af73c namespace: kserve-ci-e2e-test resourceVersion: "20048" uid: 41a7c1ff-5f2c-490e-89d9-2179fceed82a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-313c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20026" uid: 35d7d518-a351-4ddf-b07e-7a5591a42b89 kind: Event lastTimestamp: "2026-04-17T17:44:46Z" message: Scaled up replica set success-200-isvc-313c7-predictor-5bddb7798d from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:46Z" name: success-200-isvc-313c7-predictor.18a735f0ec890ece namespace: kserve-ci-e2e-test resourceVersion: "20037" uid: a8a6c4e2-8dea-4ca1-afdd-54232cb9e571 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:45:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-313c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20030" uid: aded3b5f-a712-4164-996c-778f7db98e4d kind: Event lastTimestamp: "2026-04-17T17:45:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:16Z" name: success-200-isvc-313c7-predictor.18a735f46b104f56 namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 5c7139a7-3c59-4218-8a7b-593bc2bbfcfd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:45:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-313c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20030" uid: aded3b5f-a712-4164-996c-778f7db98e4d kind: Event lastTimestamp: "2026-04-17T17:45:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:45:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:16Z" name: success-200-isvc-313c7-predictor.18a735f46b8a6201 namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: 44b8e323-0f78-4bbb-9d2f-8ee77c8fd908 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T17:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-313c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20260" uid: aded3b5f-a712-4164-996c-778f7db98e4d kind: Event lastTimestamp: "2026-04-17T17:46:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:16Z" name: success-200-isvc-313c7-predictor.18a735fb69290097 namespace: kserve-ci-e2e-test resourceVersion: "21114" uid: b84fc740-b707-4235-9d6a-9785248e6461 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T17:45:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-313c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "20260" uid: aded3b5f-a712-4164-996c-778f7db98e4d kind: Event lastTimestamp: "2026-04-17T17:46:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:16Z" name: success-200-isvc-313c7-predictor.18a735fb6978122d namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: 09f945e9-2be5-4872-b486-e9a774855e42 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:44:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-313c7 namespace: kserve-ci-e2e-test resourceVersion: "20014" uid: b99a065d-2793-4436-9000-12cc23d3fb31 kind: Event lastTimestamp: "2026-04-17T17:45:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:35Z" name: success-200-isvc-313c7.18a735f0ec494eb0 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 63da8206-f95a-4cca-bdca-f8f9398c09bd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-313c7 namespace: kserve-ci-e2e-test resourceVersion: "20014" uid: b99a065d-2793-4436-9000-12cc23d3fb31 kind: Event lastTimestamp: "2026-04-17T17:44:46Z" message: 'Failed to update status for InferenceService "success-200-isvc-313c7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-313c7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:44:46Z" name: success-200-isvc-313c7.18a735f0ed367e75 namespace: kserve-ci-e2e-test resourceVersion: "20040" uid: 14f5c7c5-570f-4f72-8a57-54d1ace54fe9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-313c7 namespace: kserve-ci-e2e-test resourceVersion: "20014" uid: b99a065d-2793-4436-9000-12cc23d3fb31 kind: Event lastTimestamp: "2026-04-17T17:44:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-313c7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:44:46Z" name: success-200-isvc-313c7.18a735f0ed36c306 namespace: kserve-ci-e2e-test resourceVersion: "20042" uid: 81b4c468-e8c1-4a35-ac0e-7c544a42c62e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-313c7 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: b99a065d-2793-4436-9000-12cc23d3fb31 kind: Event lastTimestamp: "2026-04-17T17:45:35Z" message: InferenceService [success-200-isvc-313c7] is Ready metadata: creationTimestamp: "2026-04-17T17:45:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:35Z" name: success-200-isvc-313c7.18a735fc3a66956f namespace: kserve-ci-e2e-test resourceVersion: "20596" uid: ecf85522-c40b-4fea-9a8f-62d2b7585bf7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:44:26.357438Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl namespace: kserve-ci-e2e-test resourceVersion: "19703" uid: 1aa5f184-67fd-454e-a1b7-11e64456aebe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl.18a735ec243c69b7 namespace: kserve-ci-e2e-test resourceVersion: "19712" uid: 7592117b-ebbe-41fd-8cb7-05e37fd2f76c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl namespace: kserve-ci-e2e-test resourceVersion: "19723" uid: 1aa5f184-67fd-454e-a1b7-11e64456aebe kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl.18a735ec3d9453a6 namespace: kserve-ci-e2e-test resourceVersion: "19760" uid: fcccd664-b242-42ae-bfca-78d7c2f44b57 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: 1aa5f184-67fd-454e-a1b7-11e64456aebe kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl.18a735ec3ef8e806 namespace: kserve-ci-e2e-test resourceVersion: "19762" uid: d7e466e1-dcd0-4749-826e-9aa9b16c3f91 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: 1aa5f184-67fd-454e-a1b7-11e64456aebe kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl.18a735ec422a046c namespace: kserve-ci-e2e-test resourceVersion: "19765" uid: 6e0894ed-2bd2-4186-880e-cc9256d00213 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: 1aa5f184-67fd-454e-a1b7-11e64456aebe kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl.18a735ec42a08d2c namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: 31d421b7-60b3-4013-8e8f-acfc4be13303 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: 1aa5f184-67fd-454e-a1b7-11e64456aebe kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl.18a735ec42ace292 namespace: kserve-ci-e2e-test resourceVersion: "19767" uid: 45d68ae8-15bd-46f7-bbae-b61555ffcc8f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: 1aa5f184-67fd-454e-a1b7-11e64456aebe kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl.18a735ec4609546c namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 4ec9266a-06bf-4fa6-94e4-1780b86ffa08 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: 1aa5f184-67fd-454e-a1b7-11e64456aebe kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl.18a735ec46807ba9 namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: e236be1d-7e3d-430b-b2a0-8806847291c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:44:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: 1aa5f184-67fd-454e-a1b7-11e64456aebe kind: Event lastTimestamp: "2026-04-17T17:45:04Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:04Z" name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl.18a735ec96c67c9d namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: d2d0e2a3-7ac0-487c-9964-7eaca412ff3b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: 1aa5f184-67fd-454e-a1b7-11e64456aebe kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:36Z" name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl.18a735fc799dd376 namespace: kserve-ci-e2e-test resourceVersion: "20655" uid: 8c75fae4-a2d1-45f2-8937-56e1d5575a3b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: 1aa5f184-67fd-454e-a1b7-11e64456aebe kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:36Z" name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl.18a735fc79d6a4ac namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: f8afda1c-40a5-4fc0-b353-3793bb287d65 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl namespace: kserve-ci-e2e-test resourceVersion: "19706" uid: 1aa5f184-67fd-454e-a1b7-11e64456aebe kind: Event lastTimestamp: "2026-04-17T17:45:39Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:45:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:39Z" name: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl.18a735fd1ef99589 namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: 4ffce6f9-3952-4bc1-a535-d96f3dddcfc9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a1277-predictor-79b86f6bfd namespace: kserve-ci-e2e-test resourceVersion: "19688" uid: fab9cf21-3864-47ed-aa60-23e17cc560f6 kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: 'Created pod: success-200-isvc-a1277-predictor-79b86f6bfd-4wtvl' metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-a1277-predictor-79b86f6bfd.18a735ec238fd3ed namespace: kserve-ci-e2e-test resourceVersion: "19704" uid: 8a92b7b9-1870-4f68-b9bc-ce9edee014f0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a1277-predictor namespace: kserve-ci-e2e-test resourceVersion: "19684" uid: e2c6d31f-0153-4981-af31-2ec9a70f4ea1 kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: Scaled up replica set success-200-isvc-a1277-predictor-79b86f6bfd from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-a1277-predictor.18a735ec202beb77 namespace: kserve-ci-e2e-test resourceVersion: "19693" uid: a6e8e120-7ec6-43bd-a1f8-90f37e04843d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T17:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a1277-predictor namespace: kserve-ci-e2e-test resourceVersion: "19687" uid: d3f2c2af-3d55-4d26-a16a-8b4d4914d139 kind: Event lastTimestamp: "2026-04-17T17:45:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:26Z" name: success-200-isvc-a1277-predictor.18a735ef9ea18287 namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 3420aa73-a88d-46c9-97d2-d8cc0c4bf9a7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-17T17:44:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a1277-predictor namespace: kserve-ci-e2e-test resourceVersion: "19687" uid: d3f2c2af-3d55-4d26-a16a-8b4d4914d139 kind: Event lastTimestamp: "2026-04-17T17:45:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:44:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:26Z" name: success-200-isvc-a1277-predictor.18a735ef9f237cbc namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 5fb14f0e-9eb4-4162-8a31-bc668d48659c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a1277 namespace: kserve-ci-e2e-test resourceVersion: "19670" uid: d0750fbe-1446-4389-a382-28fb322fd037 kind: Event lastTimestamp: "2026-04-17T17:45:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:14Z" name: success-200-isvc-a1277.18a735ec1fea3516 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 08108b0d-770a-4228-8aeb-e9ec973dac30 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a1277 namespace: kserve-ci-e2e-test resourceVersion: "19670" uid: d0750fbe-1446-4389-a382-28fb322fd037 kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: 'Failed to update status for InferenceService "success-200-isvc-a1277": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a1277": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-a1277.18a735ec208c6ce0 namespace: kserve-ci-e2e-test resourceVersion: "19696" uid: 5ef5080b-4899-4b14-a605-30ecb12d6f9d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a1277 namespace: kserve-ci-e2e-test resourceVersion: "19670" uid: d0750fbe-1446-4389-a382-28fb322fd037 kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a1277": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-a1277.18a735ec208cb619 namespace: kserve-ci-e2e-test resourceVersion: "19697" uid: 6d783f48-4f0e-424a-a8f4-801190f43de6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a1277 namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: d0750fbe-1446-4389-a382-28fb322fd037 kind: Event lastTimestamp: "2026-04-17T17:45:14Z" message: InferenceService [success-200-isvc-a1277] is Ready metadata: creationTimestamp: "2026-04-17T17:45:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:14Z" name: success-200-isvc-a1277.18a735f751d8467c namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 52479d61-608d-42a2-87ce-bfddfa41df0f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:55:12.089485Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5 namespace: kserve-ci-e2e-test resourceVersion: "25287" uid: c2d86f51-524c-4513-84e7-f560a56dcde3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5 to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5.18a736827cdd98ff namespace: kserve-ci-e2e-test resourceVersion: "25292" uid: b36f3a87-0d54-4206-a777-f606885072c6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5 namespace: kserve-ci-e2e-test resourceVersion: "25298" uid: c2d86f51-524c-4513-84e7-f560a56dcde3 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Add eth0 [10.132.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5.18a7368296121ce2 namespace: kserve-ci-e2e-test resourceVersion: "25352" uid: b868507f-b20c-4c46-906d-4eda5f7cfbb3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5 namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: c2d86f51-524c-4513-84e7-f560a56dcde3 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5.18a7368297807648 namespace: kserve-ci-e2e-test resourceVersion: "25354" uid: ff96176a-942e-4671-85e6-e24570291b9d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5 namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: c2d86f51-524c-4513-84e7-f560a56dcde3 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5.18a736829bae66e9 namespace: kserve-ci-e2e-test resourceVersion: "25356" uid: 34ed5b72-8a96-4831-a854-86473b83f9c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5 namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: c2d86f51-524c-4513-84e7-f560a56dcde3 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5.18a736829c1ee6fd namespace: kserve-ci-e2e-test resourceVersion: "25357" uid: d1f62c16-f4dc-4cd4-bf9c-13003e0a11e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5 namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: c2d86f51-524c-4513-84e7-f560a56dcde3 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5.18a736829c2a8e9f namespace: kserve-ci-e2e-test resourceVersion: "25358" uid: 23d65d26-fab3-4bf2-a86c-1468418cf390 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5 namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: c2d86f51-524c-4513-84e7-f560a56dcde3 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5.18a736829f70ac08 namespace: kserve-ci-e2e-test resourceVersion: "25362" uid: 51007fd1-d803-4045-bf92-b6edf64e41c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5 namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: c2d86f51-524c-4513-84e7-f560a56dcde3 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5.18a736829fe4808e namespace: kserve-ci-e2e-test resourceVersion: "25363" uid: 57d6d710-674f-4fa1-8f04-e0c404bbe8e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5 namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: c2d86f51-524c-4513-84e7-f560a56dcde3 kind: Event lastTimestamp: "2026-04-17T17:55:50Z" message: 'Readiness probe failed: dial tcp 10.132.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:50Z" name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5.18a736830ab93f4a namespace: kserve-ci-e2e-test resourceVersion: "25700" uid: 8b4025c9-82ce-4c0c-9bdb-cf8564ab3be5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T18:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5 namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: c2d86f51-524c-4513-84e7-f560a56dcde3 kind: Event lastTimestamp: "2026-04-17T18:04:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T18:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:04:26Z" name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5.18a73703a456d47f namespace: kserve-ci-e2e-test resourceVersion: "29710" uid: 9ce9890d-c9d2-4e38-b34f-41714329d9d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T18:04:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5 namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: c2d86f51-524c-4513-84e7-f560a56dcde3 kind: Event lastTimestamp: "2026-04-17T18:04:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T18:04:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:04:26Z" name: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5.18a73703a4575f1b namespace: kserve-ci-e2e-test resourceVersion: "29711" uid: fed20026-40a4-4512-856b-5c59504de47a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a935a-predictor-7c5f58bdc6 namespace: kserve-ci-e2e-test resourceVersion: "25272" uid: 6569e6d2-b75a-4bb4-a04a-eca3e94beab6 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: 'Created pod: success-200-isvc-a935a-predictor-7c5f58bdc6-7m6x5' metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-a935a-predictor-7c5f58bdc6.18a736827ca376bb namespace: kserve-ci-e2e-test resourceVersion: "25290" uid: ed60be0e-e2d1-4c50-bb9c-199b4e6822fa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a935a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25268" uid: a25dd6b0-9cd9-4369-b7f7-4ffd5ad92b12 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Scaled up replica set success-200-isvc-a935a-predictor-7c5f58bdc6 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-a935a-predictor.18a736827a96a45f namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: 5daefeda-edc0-4c6c-a85d-320ae8a2d56c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:55:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a935a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25271" uid: 4a2f75de-0e64-4f15-918b-8e849e26c568 kind: Event lastTimestamp: "2026-04-17T17:55:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:42Z" name: success-200-isvc-a935a-predictor.18a73685f9173811 namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: 2dec3ea0-1c34-4855-9283-050694d485ac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:55:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a935a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25271" uid: 4a2f75de-0e64-4f15-918b-8e849e26c568 kind: Event lastTimestamp: "2026-04-17T17:55:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:42Z" name: success-200-isvc-a935a-predictor.18a73685f9895a39 namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 8543ae82-1664-4720-8752-e13427a1c95a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:55:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a935a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25498" uid: 4a2f75de-0e64-4f15-918b-8e849e26c568 kind: Event lastTimestamp: "2026-04-17T17:56:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:27Z" name: success-200-isvc-a935a-predictor.18a7368cf72dc6cc namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: 4f6950c0-75b2-4d01-b0dc-64cb3416355f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:55:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a935a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25498" uid: 4a2f75de-0e64-4f15-918b-8e849e26c568 kind: Event lastTimestamp: "2026-04-17T17:56:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:55:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:27Z" name: success-200-isvc-a935a-predictor.18a7368cf7e11c5a namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: 9406a672-74a3-458c-b0fc-aca0d44015e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a935a namespace: kserve-ci-e2e-test resourceVersion: "25255" uid: b5113e72-56cc-4007-abb7-6853e720528f kind: Event lastTimestamp: "2026-04-17T17:56:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:56:00Z" name: success-200-isvc-a935a.18a736827a5be85f namespace: kserve-ci-e2e-test resourceVersion: "25841" uid: c2682e84-3178-4f72-ab71-8ee5046f39d6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a935a namespace: kserve-ci-e2e-test resourceVersion: "25255" uid: b5113e72-56cc-4007-abb7-6853e720528f kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-a935a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a935a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-a935a.18a736827af5f591 namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 76051531-4819-4125-aa1e-be4276edf048 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a935a namespace: kserve-ci-e2e-test resourceVersion: "25255" uid: b5113e72-56cc-4007-abb7-6853e720528f kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a935a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-a935a.18a736827af65369 namespace: kserve-ci-e2e-test resourceVersion: "25284" uid: 931758d4-4925-444d-ba6e-9d20b79d7c0a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:56:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a935a namespace: kserve-ci-e2e-test resourceVersion: "25838" uid: b5113e72-56cc-4007-abb7-6853e720528f kind: Event lastTimestamp: "2026-04-17T17:56:00Z" message: InferenceService [success-200-isvc-a935a] is Ready metadata: creationTimestamp: "2026-04-17T17:56:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:56:00Z" name: success-200-isvc-a935a.18a7368dc5a3e1fc namespace: kserve-ci-e2e-test resourceVersion: "25840" uid: b364cad0-55fa-4b5d-9ab4-ae558391eda1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:43:36.696565Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk namespace: kserve-ci-e2e-test resourceVersion: "19175" uid: 2e9c6d8c-eb46-4b9d-88bd-6f402b235473 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d1a8d-predictor-656694f686-jhmhk to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:43:36Z" name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk.18a735e09437afe6 namespace: kserve-ci-e2e-test resourceVersion: "19182" uid: c18ea564-d63b-4d5c-b6ec-3b4d501109eb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 2e9c6d8c-eb46-4b9d-88bd-6f402b235473 kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d1a8d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:36Z" name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk.18a735e0a09e5446 namespace: kserve-ci-e2e-test resourceVersion: "19212" uid: 77a7f13f-a56f-48b1-8edc-038c313250a9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk namespace: kserve-ci-e2e-test resourceVersion: "19193" uid: 2e9c6d8c-eb46-4b9d-88bd-6f402b235473 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:43:37Z" name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk.18a735e0d18ab9ae namespace: kserve-ci-e2e-test resourceVersion: "19226" uid: 81c50654-c70f-4944-be58-c7e580e13ff9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 2e9c6d8c-eb46-4b9d-88bd-6f402b235473 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:37Z" name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk.18a735e0d2e66079 namespace: kserve-ci-e2e-test resourceVersion: "19228" uid: 21cbfca1-bcf9-4734-9070-8021ed13fe6a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 2e9c6d8c-eb46-4b9d-88bd-6f402b235473 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:37Z" name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk.18a735e0d6492cff namespace: kserve-ci-e2e-test resourceVersion: "19233" uid: 96c39f5d-b8b3-4953-9918-cfb073a64666 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 2e9c6d8c-eb46-4b9d-88bd-6f402b235473 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:37Z" name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk.18a735e0d6c21592 namespace: kserve-ci-e2e-test resourceVersion: "19235" uid: c6cafc0d-a944-4d01-a68e-1757712e87a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 2e9c6d8c-eb46-4b9d-88bd-6f402b235473 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:37Z" name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk.18a735e0d6cd0855 namespace: kserve-ci-e2e-test resourceVersion: "19236" uid: 2c16e922-a8c2-47d3-af78-78c4fcd8c552 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 2e9c6d8c-eb46-4b9d-88bd-6f402b235473 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:37Z" name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk.18a735e0da286e3c namespace: kserve-ci-e2e-test resourceVersion: "19239" uid: b117a046-f3cf-4ee9-867e-6456947cec29 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 2e9c6d8c-eb46-4b9d-88bd-6f402b235473 kind: Event lastTimestamp: "2026-04-17T17:43:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:43:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:37Z" name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk.18a735e0da96be24 namespace: kserve-ci-e2e-test resourceVersion: "19241" uid: bc6c6514-bf9f-4de3-93e6-44f6186fad05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:43:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 2e9c6d8c-eb46-4b9d-88bd-6f402b235473 kind: Event lastTimestamp: "2026-04-17T17:44:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:14Z" name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk.18a735e0e8087f0a namespace: kserve-ci-e2e-test resourceVersion: "19538" uid: d226b543-9c84-4d70-b041-8adeb59f60a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 2e9c6d8c-eb46-4b9d-88bd-6f402b235473 kind: Event lastTimestamp: "2026-04-17T17:44:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:46Z" name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk.18a735f0edf751b0 namespace: kserve-ci-e2e-test resourceVersion: "20045" uid: 557cb00e-b71e-4ff0-ba24-14a6388c94e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 2e9c6d8c-eb46-4b9d-88bd-6f402b235473 kind: Event lastTimestamp: "2026-04-17T17:44:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:44:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:46Z" name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk.18a735f0edf7d478 namespace: kserve-ci-e2e-test resourceVersion: "20047" uid: b1be94d0-10e4-4b94-9f00-4f1e52c2745f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk namespace: kserve-ci-e2e-test resourceVersion: "19178" uid: 2e9c6d8c-eb46-4b9d-88bd-6f402b235473 kind: Event lastTimestamp: "2026-04-17T17:44:49Z" message: 'Readiness probe failed: Get "https://10.132.0.38:8643/healthz": dial tcp 10.132.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:44:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:49Z" name: success-200-isvc-d1a8d-predictor-656694f686-jhmhk.18a735f1701463c1 namespace: kserve-ci-e2e-test resourceVersion: "20150" uid: c36d69a1-ca97-48b4-8c84-14af61156112 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d1a8d-predictor-656694f686 namespace: kserve-ci-e2e-test resourceVersion: "19152" uid: 55e4d941-f181-43ce-a17a-9a66591480a2 kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: 'Created pod: success-200-isvc-d1a8d-predictor-656694f686-jhmhk' metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:43:36Z" name: success-200-isvc-d1a8d-predictor-656694f686.18a735e093d7e642 namespace: kserve-ci-e2e-test resourceVersion: "19179" uid: 54115602-3e69-47ff-bb98-f91b1c9ef085 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d1a8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19149" uid: 7ef95eec-c9b6-43a3-b315-e468dc6badf4 kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: Scaled up replica set success-200-isvc-d1a8d-predictor-656694f686 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:43:36Z" name: success-200-isvc-d1a8d-predictor.18a735e08fe62149 namespace: kserve-ci-e2e-test resourceVersion: "19154" uid: 7cf80987-b927-4016-b475-2dadcefc08fe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1a8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19151" uid: 57a06b79-297e-42a2-af0e-857668c6cfe4 kind: Event lastTimestamp: "2026-04-17T17:43:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:43:51Z" name: success-200-isvc-d1a8d-predictor.18a735e40f21a4ff namespace: kserve-ci-e2e-test resourceVersion: "19360" uid: 53557100-a965-48d9-acb7-43e73399d553 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1a8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19151" uid: 57a06b79-297e-42a2-af0e-857668c6cfe4 kind: Event lastTimestamp: "2026-04-17T17:43:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:43:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:43:51Z" name: success-200-isvc-d1a8d-predictor.18a735e40f7f214d namespace: kserve-ci-e2e-test resourceVersion: "19361" uid: ed14dd42-9272-45a5-a899-0ecba9715a8c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:44:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1a8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 57a06b79-297e-42a2-af0e-857668c6cfe4 kind: Event lastTimestamp: "2026-04-17T17:44:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:36Z" name: success-200-isvc-d1a8d-predictor.18a735e78e496ce0 namespace: kserve-ci-e2e-test resourceVersion: "19870" uid: b64503ac-e15f-4f1a-bc1c-4e161ea43e69 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:44:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d1a8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "19359" uid: 57a06b79-297e-42a2-af0e-857668c6cfe4 kind: Event lastTimestamp: "2026-04-17T17:44:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:44:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:44:36Z" name: success-200-isvc-d1a8d-predictor.18a735e78ebd5f76 namespace: kserve-ci-e2e-test resourceVersion: "19874" uid: 28f63658-da06-4744-9f1b-65d1034eb86f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1a8d namespace: kserve-ci-e2e-test resourceVersion: "19127" uid: ba1d0596-2eba-4b7a-aa60-37913c9836b5 kind: Event lastTimestamp: "2026-04-17T17:44:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:44:24Z" name: success-200-isvc-d1a8d.18a735e08fbe0547 namespace: kserve-ci-e2e-test resourceVersion: "19625" uid: a7042b87-bc4b-4aea-8c1d-cdaa01f237dd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1a8d namespace: kserve-ci-e2e-test resourceVersion: "19127" uid: ba1d0596-2eba-4b7a-aa60-37913c9836b5 kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: 'Failed to update status for InferenceService "success-200-isvc-d1a8d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d1a8d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:43:36Z" name: success-200-isvc-d1a8d.18a735e090139121 namespace: kserve-ci-e2e-test resourceVersion: "19156" uid: 3a070590-c79c-4547-b613-bf833c90db27 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1a8d namespace: kserve-ci-e2e-test resourceVersion: "19127" uid: ba1d0596-2eba-4b7a-aa60-37913c9836b5 kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d1a8d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:43:36Z" name: success-200-isvc-d1a8d.18a735e09013d860 namespace: kserve-ci-e2e-test resourceVersion: "19157" uid: 48b9339a-95aa-4c0c-b832-e174a3002ac7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d1a8d namespace: kserve-ci-e2e-test resourceVersion: "19623" uid: ba1d0596-2eba-4b7a-aa60-37913c9836b5 kind: Event lastTimestamp: "2026-04-17T17:44:24Z" message: InferenceService [success-200-isvc-d1a8d] is Ready metadata: creationTimestamp: "2026-04-17T17:44:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:44:24Z" name: success-200-isvc-d1a8d.18a735eba63d371e namespace: kserve-ci-e2e-test resourceVersion: "19624" uid: e0e3e058-567d-4023-a4a7-4bb885d188b8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:34:21.624527Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: 7501e586-6f33-437c-a1de-8e37d3a78b56 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d987b-predictor-7847c5c48b-q65fm to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:34:21Z" name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm.18a7355f57588b03 namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 7f73f8d9-0c91-49f6-85b7-077d1efa8714 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 7501e586-6f33-437c-a1de-8e37d3a78b56 kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d987b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:21Z" name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm.18a7355f615be75f namespace: kserve-ci-e2e-test resourceVersion: "14825" uid: 2f1d369d-6b44-4ee0-81ba-21116812eb44 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: 7501e586-6f33-437c-a1de-8e37d3a78b56 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:34:22Z" name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm.18a7355f94721761 namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: e885aafb-a757-4cfc-931d-11e7160d732e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 7501e586-6f33-437c-a1de-8e37d3a78b56 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm.18a7355f95c600d1 namespace: kserve-ci-e2e-test resourceVersion: "14854" uid: 67587b7c-454b-4738-b4a8-f494913ee3a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 7501e586-6f33-437c-a1de-8e37d3a78b56 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm.18a7355f9a08d3a0 namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: 5553bde3-64f2-424f-a23d-86e7487fb54a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 7501e586-6f33-437c-a1de-8e37d3a78b56 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm.18a7355f9a77448b namespace: kserve-ci-e2e-test resourceVersion: "14859" uid: 887b0801-05e1-4075-90f6-340fd82221f0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 7501e586-6f33-437c-a1de-8e37d3a78b56 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm.18a7355f9a841398 namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: aa038c1f-65a1-4b58-aa1a-a4b843788b08 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 7501e586-6f33-437c-a1de-8e37d3a78b56 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm.18a7355f9ddbbd09 namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: 9db21ba2-85fe-4738-ba9b-e002b44394aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 7501e586-6f33-437c-a1de-8e37d3a78b56 kind: Event lastTimestamp: "2026-04-17T17:34:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:34:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:22Z" name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm.18a7355f9e4e039b namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: 5d6068e7-37dc-4dbf-8913-209dab9641d7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-17T17:34:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 7501e586-6f33-437c-a1de-8e37d3a78b56 kind: Event lastTimestamp: "2026-04-17T17:43:39Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:34:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:39Z" name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm.18a7355fb678b769 namespace: kserve-ci-e2e-test resourceVersion: "19262" uid: 024c23f0-179d-4d04-b5e9-20f5a3239920 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 7501e586-6f33-437c-a1de-8e37d3a78b56 kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:36Z" name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm.18a735e08eb6a213 namespace: kserve-ci-e2e-test resourceVersion: "19146" uid: cf250e17-2b45-4f87-abcc-caf17fff906a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 7501e586-6f33-437c-a1de-8e37d3a78b56 kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:36Z" name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm.18a735e08eb71664 namespace: kserve-ci-e2e-test resourceVersion: "19150" uid: 4b4cc927-394c-4a49-ab8e-744539f54fd7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: 7501e586-6f33-437c-a1de-8e37d3a78b56 kind: Event lastTimestamp: "2026-04-17T17:43:39Z" message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": dial tcp 10.132.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:39Z" name: success-200-isvc-d987b-predictor-7847c5c48b-q65fm.18a735e12ae499af namespace: kserve-ci-e2e-test resourceVersion: "19261" uid: 0e847847-b5c5-4a8d-a598-b664e2fcf89e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d987b-predictor-7847c5c48b namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 2846bf29-88b4-46f1-8517-f13213e1d8d3 kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: 'Created pod: success-200-isvc-d987b-predictor-7847c5c48b-q65fm' metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:21Z" name: success-200-isvc-d987b-predictor-7847c5c48b.18a7355f56f0cff0 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: 2d2ebf0a-5fe6-463a-a6ff-4441053739f2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d987b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: f8977dbd-f6be-40f9-a72a-2163201b9332 kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: Scaled up replica set success-200-isvc-d987b-predictor-7847c5c48b from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:21Z" name: success-200-isvc-d987b-predictor.18a7355f5528bd97 namespace: kserve-ci-e2e-test resourceVersion: "14768" uid: 88d9bb7c-742b-47c1-a56d-4259af8226b1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:34:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d987b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: a6a7a752-2662-41bd-837e-bcaa6db37d72 kind: Event lastTimestamp: "2026-04-17T17:34:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:51Z" name: success-200-isvc-d987b-predictor.18a73562d3cbaa67 namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: ff19e538-dc78-4267-80fa-88e94c30033d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:34:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d987b-predictor namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: a6a7a752-2662-41bd-837e-bcaa6db37d72 kind: Event lastTimestamp: "2026-04-17T17:34:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:51Z" name: success-200-isvc-d987b-predictor.18a73562d43455d7 namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: 3c26ca3d-9bc2-4df1-a1f5-ab4b2215191f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:35:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d987b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: a6a7a752-2662-41bd-837e-bcaa6db37d72 kind: Event lastTimestamp: "2026-04-17T17:35:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:21Z" name: success-200-isvc-d987b-predictor.18a73569d1bc917b namespace: kserve-ci-e2e-test resourceVersion: "15614" uid: 107e9736-e391-41f8-82a0-0d791343e52b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:35:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d987b-predictor namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: a6a7a752-2662-41bd-837e-bcaa6db37d72 kind: Event lastTimestamp: "2026-04-17T17:35:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:35:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:21Z" name: success-200-isvc-d987b-predictor.18a73569d20bd45e namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: 95917ba5-624d-44e0-a0fd-d01e9657194d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d987b namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 13382e20-d2e7-4c3c-8798-bdb142149f61 kind: Event lastTimestamp: "2026-04-17T17:35:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:35:09Z" name: success-200-isvc-d987b.18a7355f55247dae namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: 88894afe-c18f-4870-9c10-bf933d172449 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d987b namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 13382e20-d2e7-4c3c-8798-bdb142149f61 kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: 'Failed to update status for InferenceService "success-200-isvc-d987b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d987b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:34:21Z" name: success-200-isvc-d987b.18a7355f55f5337b namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: 93f971de-3ba5-4dea-8724-01c3ff1e03c5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d987b namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 13382e20-d2e7-4c3c-8798-bdb142149f61 kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d987b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:34:21Z" name: success-200-isvc-d987b.18a7355f55f59537 namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: a81b8abc-7eab-413b-8403-e999c699c94a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d987b namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 13382e20-d2e7-4c3c-8798-bdb142149f61 kind: Event lastTimestamp: "2026-04-17T17:35:09Z" message: InferenceService [success-200-isvc-d987b] is Ready metadata: creationTimestamp: "2026-04-17T17:35:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:35:09Z" name: success-200-isvc-d987b.18a7356a71866ecb namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 28b9592e-fbe5-4b40-ac73-29b64ff90398 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:45:36.532002Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj namespace: kserve-ci-e2e-test resourceVersion: "20656" uid: b4a954bf-fc87-4e85-a790-dc1e57f4e5a5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:45:36Z" name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj.18a735fc7af75991 namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: d3696c4e-c38c-43d6-874f-739bb7095a1d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: b4a954bf-fc87-4e85-a790-dc1e57f4e5a5 kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:45:36Z" name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj.18a735fc94713daa namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 846bda82-70ba-4daf-b696-e7566076415a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: b4a954bf-fc87-4e85-a790-dc1e57f4e5a5 kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:36Z" name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj.18a735fc95f0f7f8 namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: 20263b20-3aca-4c6e-b7d4-5bfdf3b68f1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: b4a954bf-fc87-4e85-a790-dc1e57f4e5a5 kind: Event lastTimestamp: "2026-04-17T17:45:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:37Z" name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj.18a735fc98ec99bd namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: 6378b336-240f-49ff-9796-c91e7aa6152f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: b4a954bf-fc87-4e85-a790-dc1e57f4e5a5 kind: Event lastTimestamp: "2026-04-17T17:45:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:37Z" name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj.18a735fc99674f65 namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: c2a98cbb-fdfb-4c80-9c8a-8a746233a80a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: b4a954bf-fc87-4e85-a790-dc1e57f4e5a5 kind: Event lastTimestamp: "2026-04-17T17:45:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:37Z" name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj.18a735fc99743cac namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 0f8813fb-bb95-4d6b-bb09-70fef1345782 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: b4a954bf-fc87-4e85-a790-dc1e57f4e5a5 kind: Event lastTimestamp: "2026-04-17T17:45:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:37Z" name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj.18a735fc9c53d69a namespace: kserve-ci-e2e-test resourceVersion: "20730" uid: 3ad49624-a076-4d2d-aca9-71b0db4080de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: b4a954bf-fc87-4e85-a790-dc1e57f4e5a5 kind: Event lastTimestamp: "2026-04-17T17:45:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:45:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:45:37Z" name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj.18a735fc9ccb28c1 namespace: kserve-ci-e2e-test resourceVersion: "20731" uid: 406d3894-d823-4e80-a4c5-64104293a2c6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:45:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: b4a954bf-fc87-4e85-a790-dc1e57f4e5a5 kind: Event lastTimestamp: "2026-04-17T17:46:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:45:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:46:14Z" name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj.18a735fcf21e272f namespace: kserve-ci-e2e-test resourceVersion: "21096" uid: e3bb0718-e665-4958-bcfa-3397a8187a6e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: b4a954bf-fc87-4e85-a790-dc1e57f4e5a5 kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:51Z" name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj.18a7367da62d5e67 namespace: kserve-ci-e2e-test resourceVersion: "24928" uid: ffd13d73-a022-44b6-8b82-a4156a172082 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: b4a954bf-fc87-4e85-a790-dc1e57f4e5a5 kind: Event lastTimestamp: "2026-04-17T17:54:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:54:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:51Z" name: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj.18a7367da62db768 namespace: kserve-ci-e2e-test resourceVersion: "24930" uid: 372f93f4-f01f-4b3d-9411-b0b60f9b7ca1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-dd512-predictor-7d56f7b46b namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 74b3e435-73e4-409d-9b37-a476792cef62 kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: 'Created pod: success-200-isvc-dd512-predictor-7d56f7b46b-t6mqj' metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:36Z" name: success-200-isvc-dd512-predictor-7d56f7b46b.18a735fc7a79c439 namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 76e8b5b4-4ec3-4ce4-817c-9e3ee8f0b879 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-dd512-predictor namespace: kserve-ci-e2e-test resourceVersion: "20640" uid: e705b01c-9f0c-4437-8d14-794f086c09ff kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: Scaled up replica set success-200-isvc-dd512-predictor-7d56f7b46b from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:45:36Z" name: success-200-isvc-dd512-predictor.18a735fc7834e61b namespace: kserve-ci-e2e-test resourceVersion: "20649" uid: 4cc851f6-7794-476e-bc09-c5b1cd3edc10 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:45:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dd512-predictor namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: b669a2df-8a21-48ac-af46-136c8e76eccb kind: Event lastTimestamp: "2026-04-17T17:46:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:06Z" name: success-200-isvc-dd512-predictor.18a735fff72d4c22 namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 0cf4138f-95c1-4580-ad56-0f16744a4d7d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:45:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dd512-predictor namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: b669a2df-8a21-48ac-af46-136c8e76eccb kind: Event lastTimestamp: "2026-04-17T17:46:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:06Z" name: success-200-isvc-dd512-predictor.18a735fff78fdba8 namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: e5ad49e2-8bce-4cab-b629-3a3aa98d4957 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:46:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dd512-predictor namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: b669a2df-8a21-48ac-af46-136c8e76eccb kind: Event lastTimestamp: "2026-04-17T17:46:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:51Z" name: success-200-isvc-dd512-predictor.18a73606f633ec20 namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: 42382c51-8eaf-4812-9f79-306ad9a284e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:46:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dd512-predictor namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: b669a2df-8a21-48ac-af46-136c8e76eccb kind: Event lastTimestamp: "2026-04-17T17:46:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:46:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:46:51Z" name: success-200-isvc-dd512-predictor.18a73606f6a9c9aa namespace: kserve-ci-e2e-test resourceVersion: "21430" uid: a8f6d4d5-504c-4b80-8eb6-4364b44909d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dd512 namespace: kserve-ci-e2e-test resourceVersion: "20625" uid: ddbe375c-821d-40ea-99ba-dae4efca4fe9 kind: Event lastTimestamp: "2026-04-17T17:46:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:46:24Z" name: success-200-isvc-dd512.18a735fc781076ae namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 7f10bcb5-3d76-4f87-80a4-4b1b4e346cf2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dd512 namespace: kserve-ci-e2e-test resourceVersion: "20625" uid: ddbe375c-821d-40ea-99ba-dae4efca4fe9 kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: 'Failed to update status for InferenceService "success-200-isvc-dd512": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-dd512": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:36Z" name: success-200-isvc-dd512.18a735fc78b982dd namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: 80d68db3-85b8-4ffb-bf87-8ba6e7422ffd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:45:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dd512 namespace: kserve-ci-e2e-test resourceVersion: "20625" uid: ddbe375c-821d-40ea-99ba-dae4efca4fe9 kind: Event lastTimestamp: "2026-04-17T17:45:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-dd512": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:45:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:45:36Z" name: success-200-isvc-dd512.18a735fc78b9cbc5 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 8f57c84f-bf66-426e-b489-07e5df481e52 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:46:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dd512 namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: ddbe375c-821d-40ea-99ba-dae4efca4fe9 kind: Event lastTimestamp: "2026-04-17T17:46:24Z" message: InferenceService [success-200-isvc-dd512] is Ready metadata: creationTimestamp: "2026-04-17T17:46:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:46:24Z" name: success-200-isvc-dd512.18a73607ad58551d namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: e7441ab2-3c74-4c58-a0fd-55f329c56a13 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:54:01.852845Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj namespace: kserve-ci-e2e-test resourceVersion: "24418" uid: 531417bf-dd53-4252-8b1b-81c2027ff5c6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e4a99-predictor-58f48cbb54-d9szj to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:54:01Z" name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj.18a73672226f8053 namespace: kserve-ci-e2e-test resourceVersion: "24428" uid: 87c34339-5cf8-4d5e-9c8f-142be8df8f3b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj namespace: kserve-ci-e2e-test resourceVersion: "24423" uid: 531417bf-dd53-4252-8b1b-81c2027ff5c6 kind: Event lastTimestamp: "2026-04-17T17:54:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e4a99-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:02Z" name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj.18a736722dd1373a namespace: kserve-ci-e2e-test resourceVersion: "24457" uid: edcf3d2f-da94-421d-9e40-3bae412b8a1c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj namespace: kserve-ci-e2e-test resourceVersion: "24438" uid: 531417bf-dd53-4252-8b1b-81c2027ff5c6 kind: Event lastTimestamp: "2026-04-17T17:54:02Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:54:02Z" name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj.18a736725fc8e306 namespace: kserve-ci-e2e-test resourceVersion: "24479" uid: f9ec94ed-fe09-4f55-ad01-47e53bd5a10d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj namespace: kserve-ci-e2e-test resourceVersion: "24423" uid: 531417bf-dd53-4252-8b1b-81c2027ff5c6 kind: Event lastTimestamp: "2026-04-17T17:54:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:02Z" name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj.18a736726128edb3 namespace: kserve-ci-e2e-test resourceVersion: "24482" uid: 7f2f942d-a987-4731-975a-fa27218985b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj namespace: kserve-ci-e2e-test resourceVersion: "24423" uid: 531417bf-dd53-4252-8b1b-81c2027ff5c6 kind: Event lastTimestamp: "2026-04-17T17:54:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:02Z" name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj.18a7367263fdad56 namespace: kserve-ci-e2e-test resourceVersion: "24484" uid: ce029a63-9ed9-4ab9-8d24-2392e2d96caf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj namespace: kserve-ci-e2e-test resourceVersion: "24423" uid: 531417bf-dd53-4252-8b1b-81c2027ff5c6 kind: Event lastTimestamp: "2026-04-17T17:54:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:02Z" name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj.18a73672646f8772 namespace: kserve-ci-e2e-test resourceVersion: "24485" uid: c25021f7-aad3-4dee-b3db-e3bd7829e593 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj namespace: kserve-ci-e2e-test resourceVersion: "24423" uid: 531417bf-dd53-4252-8b1b-81c2027ff5c6 kind: Event lastTimestamp: "2026-04-17T17:54:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:54:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:02Z" name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj.18a73672647d4c81 namespace: kserve-ci-e2e-test resourceVersion: "24486" uid: bcaae9d4-7003-475f-ab03-047b4f071efa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj namespace: kserve-ci-e2e-test resourceVersion: "24423" uid: 531417bf-dd53-4252-8b1b-81c2027ff5c6 kind: Event lastTimestamp: "2026-04-17T17:54:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:03Z" name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj.18a736726762d121 namespace: kserve-ci-e2e-test resourceVersion: "24487" uid: 9e1b7890-fd60-499c-929d-a163b9fef2ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj namespace: kserve-ci-e2e-test resourceVersion: "24423" uid: 531417bf-dd53-4252-8b1b-81c2027ff5c6 kind: Event lastTimestamp: "2026-04-17T17:54:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:03Z" name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj.18a7367267cf3300 namespace: kserve-ci-e2e-test resourceVersion: "24488" uid: 535591f5-6f5e-4696-aea8-1ec21bcce472 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:54:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj namespace: kserve-ci-e2e-test resourceVersion: "24423" uid: 531417bf-dd53-4252-8b1b-81c2027ff5c6 kind: Event lastTimestamp: "2026-04-17T17:54:39Z" message: 'Readiness probe failed: dial tcp 10.132.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:54:39Z" name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj.18a736727399e6d8 namespace: kserve-ci-e2e-test resourceVersion: "24774" uid: 62b122b2-3e63-47d0-b047-03abaeafdd5d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj namespace: kserve-ci-e2e-test resourceVersion: "24423" uid: 531417bf-dd53-4252-8b1b-81c2027ff5c6 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj.18a736827bcfa126 namespace: kserve-ci-e2e-test resourceVersion: "25286" uid: 136b2169-834c-45f8-b6f6-dcac0e3a7aaf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj namespace: kserve-ci-e2e-test resourceVersion: "24423" uid: 531417bf-dd53-4252-8b1b-81c2027ff5c6 kind: Event lastTimestamp: "2026-04-17T17:55:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:55:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:12Z" name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj.18a736827bd07c47 namespace: kserve-ci-e2e-test resourceVersion: "25288" uid: d531c82a-6dd6-4126-b2d0-a27ae8f129da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj namespace: kserve-ci-e2e-test resourceVersion: "24423" uid: 531417bf-dd53-4252-8b1b-81c2027ff5c6 kind: Event lastTimestamp: "2026-04-17T17:55:14Z" message: 'Readiness probe failed: Get "https://10.132.0.46:8643/healthz": dial tcp 10.132.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:14Z" name: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj.18a73682fbb55895 namespace: kserve-ci-e2e-test resourceVersion: "25389" uid: 855593aa-85e0-424a-b8cd-8f9834bd06f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e4a99-predictor-58f48cbb54 namespace: kserve-ci-e2e-test resourceVersion: "24394" uid: 8d38b185-09cc-4fd7-b319-e4a7e5305a65 kind: Event lastTimestamp: "2026-04-17T17:54:01Z" message: 'Created pod: success-200-isvc-e4a99-predictor-58f48cbb54-d9szj' metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:54:01Z" name: success-200-isvc-e4a99-predictor-58f48cbb54.18a736722201d5f7 namespace: kserve-ci-e2e-test resourceVersion: "24422" uid: 416df9ba-2301-4581-9477-7eae20681b98 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e4a99-predictor namespace: kserve-ci-e2e-test resourceVersion: "24390" uid: 2acdb3a4-f0d5-4094-b501-db05de1a5b52 kind: Event lastTimestamp: "2026-04-17T17:54:01Z" message: Scaled up replica set success-200-isvc-e4a99-predictor-58f48cbb54 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:54:01Z" name: success-200-isvc-e4a99-predictor.18a736721e411620 namespace: kserve-ci-e2e-test resourceVersion: "24396" uid: c1c6456d-6a7a-4e94-9501-1ae3fe1e5544 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:54:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4a99-predictor namespace: kserve-ci-e2e-test resourceVersion: "24392" uid: 400d58cb-17b4-4f7d-93c4-f23efec535ea kind: Event lastTimestamp: "2026-04-17T17:54:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:54:31Z" name: success-200-isvc-e4a99-predictor.18a736759ca1596a namespace: kserve-ci-e2e-test resourceVersion: "24718" uid: e90226a4-d83d-42bb-85a1-84c9a245af9e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:54:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4a99-predictor namespace: kserve-ci-e2e-test resourceVersion: "24392" uid: 400d58cb-17b4-4f7d-93c4-f23efec535ea kind: Event lastTimestamp: "2026-04-17T17:54:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:54:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:54:31Z" name: success-200-isvc-e4a99-predictor.18a736759d12e4e7 namespace: kserve-ci-e2e-test resourceVersion: "24720" uid: bd49fc12-8ae3-4b3a-b81f-a2452c35a248 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:54:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4a99-predictor namespace: kserve-ci-e2e-test resourceVersion: "24601" uid: 400d58cb-17b4-4f7d-93c4-f23efec535ea kind: Event lastTimestamp: "2026-04-17T17:55:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:01Z" name: success-200-isvc-e4a99-predictor.18a7367c9a80364b namespace: kserve-ci-e2e-test resourceVersion: "25115" uid: 2b2c6304-4399-42ae-84d7-915aa9354809 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:54:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e4a99-predictor namespace: kserve-ci-e2e-test resourceVersion: "24601" uid: 400d58cb-17b4-4f7d-93c4-f23efec535ea kind: Event lastTimestamp: "2026-04-17T17:55:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:54:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:01Z" name: success-200-isvc-e4a99-predictor.18a7367c9aeb6740 namespace: kserve-ci-e2e-test resourceVersion: "25117" uid: 1907cb74-dece-4f7e-a679-fbca5c065a52 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4a99 namespace: kserve-ci-e2e-test resourceVersion: "24370" uid: c2d40790-da31-405a-8184-549bb39de29f kind: Event lastTimestamp: "2026-04-17T17:54:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:54:49Z" name: success-200-isvc-e4a99.18a736721deda6f2 namespace: kserve-ci-e2e-test resourceVersion: "24862" uid: e0201840-5cde-4392-8ff1-fe7cba6ec501 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4a99 namespace: kserve-ci-e2e-test resourceVersion: "24370" uid: c2d40790-da31-405a-8184-549bb39de29f kind: Event lastTimestamp: "2026-04-17T17:54:01Z" message: 'Failed to update status for InferenceService "success-200-isvc-e4a99": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e4a99": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:54:01Z" name: success-200-isvc-e4a99.18a736721e4d29f3 namespace: kserve-ci-e2e-test resourceVersion: "24398" uid: 89c23509-545e-4d63-9663-afbc56cf9e82 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4a99 namespace: kserve-ci-e2e-test resourceVersion: "24370" uid: c2d40790-da31-405a-8184-549bb39de29f kind: Event lastTimestamp: "2026-04-17T17:54:01Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e4a99": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:54:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:54:01Z" name: success-200-isvc-e4a99.18a736721e4d7261 namespace: kserve-ci-e2e-test resourceVersion: "24400" uid: 29657171-7653-4930-91bd-974bc238ce09 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:54:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e4a99 namespace: kserve-ci-e2e-test resourceVersion: "24859" uid: c2d40790-da31-405a-8184-549bb39de29f kind: Event lastTimestamp: "2026-04-17T17:54:49Z" message: InferenceService [success-200-isvc-e4a99] is Ready metadata: creationTimestamp: "2026-04-17T17:54:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:54:49Z" name: success-200-isvc-e4a99.18a7367d2e7b71f9 namespace: kserve-ci-e2e-test resourceVersion: "24860" uid: 22cac43f-a2f8-45f0-b248-49d7727146be reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:35:11.469200Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb namespace: kserve-ci-e2e-test resourceVersion: "15401" uid: 4ca08d1c-47e9-4764-9175-f10bfaf98bd4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:35:11Z" name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb.18a7356af251da28 namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 5eaf4d82-f8c7-4578-bf3c-c84d3506eee1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 4ca08d1c-47e9-4764-9175-f10bfaf98bd4 kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-fa93d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:11Z" name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb.18a7356afe209aec namespace: kserve-ci-e2e-test resourceVersion: "15471" uid: ce6ae142-2e1f-485a-961a-f646b18114f7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 4ca08d1c-47e9-4764-9175-f10bfaf98bd4 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:35:12Z" name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb.18a7356b2f40f2f5 namespace: kserve-ci-e2e-test resourceVersion: "15488" uid: 9fbcdc86-0347-4289-8998-64b2dc0d4bc2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 4ca08d1c-47e9-4764-9175-f10bfaf98bd4 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:12Z" name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb.18a7356b30cbca1e namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: 415a0981-86e5-41dc-a240-ae7bd41368a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 4ca08d1c-47e9-4764-9175-f10bfaf98bd4 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:12Z" name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb.18a7356b346c6b7a namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 2fdb17f9-3958-4930-8bca-3f3aed0ab667 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 4ca08d1c-47e9-4764-9175-f10bfaf98bd4 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:12Z" name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb.18a7356b34e0b3ef namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: 9753e767-aaca-468a-b751-02331497297f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 4ca08d1c-47e9-4764-9175-f10bfaf98bd4 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:12Z" name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb.18a7356b34ebf736 namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: 25da3d0d-1e41-4438-8387-9f03ea55c126 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 4ca08d1c-47e9-4764-9175-f10bfaf98bd4 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:12Z" name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb.18a7356b38c7cae8 namespace: kserve-ci-e2e-test resourceVersion: "15499" uid: 77de02ec-cce8-489a-b9e9-128800c4c75a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 4ca08d1c-47e9-4764-9175-f10bfaf98bd4 kind: Event lastTimestamp: "2026-04-17T17:35:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:12Z" name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb.18a7356b393e46b5 namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 041c7535-f561-4764-91a8-01ce92e579c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:35:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 4ca08d1c-47e9-4764-9175-f10bfaf98bd4 kind: Event lastTimestamp: "2026-04-17T17:35:50Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-17T17:35:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:50Z" name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb.18a7356ba17cf65e namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: 7a0fac09-fd0c-43af-8063-58d1609efd93 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 4ca08d1c-47e9-4764-9175-f10bfaf98bd4 kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb.18a735ec218325a1 namespace: kserve-ci-e2e-test resourceVersion: "19698" uid: 2d198ebe-a822-4efc-a94c-83733826c96a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:44:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 4ca08d1c-47e9-4764-9175-f10bfaf98bd4 kind: Event lastTimestamp: "2026-04-17T17:44:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-17T17:44:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:26Z" name: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb.18a735ec21841a16 namespace: kserve-ci-e2e-test resourceVersion: "19700" uid: c2c60f3a-02ac-4b5a-8655-312ec421db68 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fa93d-predictor-6cc78c6bf6 namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: a4c92b75-e918-44cf-892c-119d74a4aa54 kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: 'Created pod: success-200-isvc-fa93d-predictor-6cc78c6bf6-t78gb' metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:11Z" name: success-200-isvc-fa93d-predictor-6cc78c6bf6.18a7356af1e0eed2 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: ccfb8e34-f65c-4848-bca0-f99f0e403a1e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fa93d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: acef6127-d793-4731-8565-8d3295cd38dd kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: Scaled up replica set success-200-isvc-fa93d-predictor-6cc78c6bf6 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:11Z" name: success-200-isvc-fa93d-predictor.18a7356af000e86b namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: 9baabbc0-36d1-4eda-a175-c6241df9d56f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:35:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fa93d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: 9c22b562-cc28-46ec-b4bc-08b095ad71ee kind: Event lastTimestamp: "2026-04-17T17:35:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:56Z" name: success-200-isvc-fa93d-predictor.18a7356e6e944154 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 221b0dac-db9e-46f6-a38d-2a3a8f33347a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-17T17:35:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fa93d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: 9c22b562-cc28-46ec-b4bc-08b095ad71ee kind: Event lastTimestamp: "2026-04-17T17:35:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:56Z" name: success-200-isvc-fa93d-predictor.18a7356e6f00ddfa namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 82ef5cba-2a1b-49cb-8bd1-e0e82d6849d4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fa93d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 9c22b562-cc28-46ec-b4bc-08b095ad71ee kind: Event lastTimestamp: "2026-04-17T17:36:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:36:26Z" name: success-200-isvc-fa93d-predictor.18a73578ebd1c9d4 namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: 8990e3e6-de2c-495b-88b6-45f18cf5415d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:36:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fa93d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 9c22b562-cc28-46ec-b4bc-08b095ad71ee kind: Event lastTimestamp: "2026-04-17T17:36:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-17T17:36:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:36:26Z" name: success-200-isvc-fa93d-predictor.18a73578ec217fb6 namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: f4c49e7f-f82a-45d2-bdef-9a87fa545836 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fa93d namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 8bfa7170-1c04-464d-8f85-fbb407c89dac kind: Event lastTimestamp: "2026-04-17T17:36:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:36:00Z" name: success-200-isvc-fa93d.18a7356aefd7d28c namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: 39eecb87-84f5-40c6-9f79-4a4b9c5ca3f1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fa93d namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 8bfa7170-1c04-464d-8f85-fbb407c89dac kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: 'Failed to update status for InferenceService "success-200-isvc-fa93d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fa93d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:35:11Z" name: success-200-isvc-fa93d.18a7356af03ac051 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: 704eb822-96be-4db4-a5a4-851d1ebc7029 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fa93d namespace: kserve-ci-e2e-test resourceVersion: "15369" uid: 8bfa7170-1c04-464d-8f85-fbb407c89dac kind: Event lastTimestamp: "2026-04-17T17:35:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fa93d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-17T17:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:35:11Z" name: success-200-isvc-fa93d.18a7356af03b1526 namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: d7ff5983-4537-407b-8e0f-d2962bd5290b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:36:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fa93d namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 8bfa7170-1c04-464d-8f85-fbb407c89dac kind: Event lastTimestamp: "2026-04-17T17:36:00Z" message: InferenceService [success-200-isvc-fa93d] is Ready metadata: creationTimestamp: "2026-04-17T17:36:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:36:00Z" name: success-200-isvc-fa93d.18a735765c850697 namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: bee8521e-d885-47a9-a58b-47a73056fbf6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:55:51.406567Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0283f-7dfb866856-9q6fw namespace: kserve-ci-e2e-test resourceVersion: "25721" uid: 7b1804bb-494c-452d-9ab6-c616587d7f91 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-0283f-7dfb866856-9q6fw to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:55:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:55:51Z" name: switch-graph-0283f-7dfb866856-9q6fw.18a7368ba458acea namespace: kserve-ci-e2e-test resourceVersion: "25726" uid: c80eed10-7f06-4328-8595-a85c3620c2aa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:51Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0283f-7dfb866856-9q6fw namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: 7b1804bb-494c-452d-9ab6-c616587d7f91 kind: Event lastTimestamp: "2026-04-17T17:55:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-0283f-serving-cert" not found' metadata: creationTimestamp: "2026-04-17T17:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:51Z" name: switch-graph-0283f-7dfb866856-9q6fw.18a7368bb27f7ef3 namespace: kserve-ci-e2e-test resourceVersion: "25740" uid: 32e1bdc3-b829-4357-b841-07dd0d20ea43 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:52Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0283f-7dfb866856-9q6fw namespace: kserve-ci-e2e-test resourceVersion: "25732" uid: 7b1804bb-494c-452d-9ab6-c616587d7f91 kind: Event lastTimestamp: "2026-04-17T17:55:52Z" message: Add eth0 [10.132.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:55:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:55:52Z" name: switch-graph-0283f-7dfb866856-9q6fw.18a7368be1330a20 namespace: kserve-ci-e2e-test resourceVersion: "25749" uid: d75a5692-2658-4ae9-8a6f-d63b523b0a56 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0283f} kind: Pod name: switch-graph-0283f-7dfb866856-9q6fw namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: 7b1804bb-494c-452d-9ab6-c616587d7f91 kind: Event lastTimestamp: "2026-04-17T17:55:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0589637b57a3315022b8ebf7fc1b0f9831e3a45e07a3fc2fab24137dd8f190f9" already present on machine metadata: creationTimestamp: "2026-04-17T17:55:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:52Z" name: switch-graph-0283f-7dfb866856-9q6fw.18a7368be26a72da namespace: kserve-ci-e2e-test resourceVersion: "25751" uid: b1c1c2e8-596a-4133-a6eb-e4b85d7967b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0283f} kind: Pod name: switch-graph-0283f-7dfb866856-9q6fw namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: 7b1804bb-494c-452d-9ab6-c616587d7f91 kind: Event lastTimestamp: "2026-04-17T17:55:52Z" message: 'Created container: switch-graph-0283f' metadata: creationTimestamp: "2026-04-17T17:55:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:52Z" name: switch-graph-0283f-7dfb866856-9q6fw.18a7368be51bc49b namespace: kserve-ci-e2e-test resourceVersion: "25752" uid: 3e6bba06-1f28-4ed1-b5ce-20a21dbfe412 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0283f} kind: Pod name: switch-graph-0283f-7dfb866856-9q6fw namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: 7b1804bb-494c-452d-9ab6-c616587d7f91 kind: Event lastTimestamp: "2026-04-17T17:55:52Z" message: Started container switch-graph-0283f metadata: creationTimestamp: "2026-04-17T17:55:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:55:52Z" name: switch-graph-0283f-7dfb866856-9q6fw.18a7368be59df2e9 namespace: kserve-ci-e2e-test resourceVersion: "25754" uid: 7779ba39-9905-4884-a0bb-c0fd53b05689 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T18:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0283f} kind: Pod name: switch-graph-0283f-7dfb866856-9q6fw namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: 7b1804bb-494c-452d-9ab6-c616587d7f91 kind: Event lastTimestamp: "2026-04-17T18:12:10Z" message: Stopping container switch-graph-0283f metadata: creationTimestamp: "2026-04-17T18:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:12:10Z" name: switch-graph-0283f-7dfb866856-9q6fw.18a7376faccb80b9 namespace: kserve-ci-e2e-test resourceVersion: "32910" uid: 3a6e9f5f-e4ee-4da9-ab57-53c8b87f1176 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-17T18:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0283f} kind: Pod name: switch-graph-0283f-7dfb866856-9q6fw namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: 7b1804bb-494c-452d-9ab6-c616587d7f91 kind: Event lastTimestamp: "2026-04-17T18:12:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T18:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T18:12:33Z" name: switch-graph-0283f-7dfb866856-9q6fw.18a73770541b16e5 namespace: kserve-ci-e2e-test resourceVersion: "33147" uid: 0c3d15a8-d27c-411d-8418-dbdd62287ce1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-0283f-7dfb866856 namespace: kserve-ci-e2e-test resourceVersion: "25718" uid: 4a7e6b58-0e54-4ae2-bca1-0f4f8d61dfdf kind: Event lastTimestamp: "2026-04-17T17:55:51Z" message: 'Created pod: switch-graph-0283f-7dfb866856-9q6fw' metadata: creationTimestamp: "2026-04-17T17:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:51Z" name: switch-graph-0283f-7dfb866856.18a7368ba3ef8ca4 namespace: kserve-ci-e2e-test resourceVersion: "25722" uid: 7887d8d9-1382-4e75-b427-4c0b28f59a2c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-0283f namespace: kserve-ci-e2e-test resourceVersion: "25716" uid: b3776cc2-a154-4123-aa4b-30a8e6d53b04 kind: Event lastTimestamp: "2026-04-17T17:55:51Z" message: Scaled up replica set switch-graph-0283f-7dfb866856 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:55:51Z" name: switch-graph-0283f.18a7368ba314a031 namespace: kserve-ci-e2e-test resourceVersion: "25719" uid: ce218212-8825-4e7e-a83a-ab8d2ae8eab6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:55:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-0283f namespace: kserve-ci-e2e-test resourceVersion: "25734" uid: 3a4525d5-e943-406f-bc19-6b4169b22507 kind: Event lastTimestamp: "2026-04-17T17:55:51Z" message: InferenceGraph [switch-graph-0283f] is Ready metadata: creationTimestamp: "2026-04-17T17:55:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:55:51Z" name: switch-graph-0283f.18a7368ba72df49d namespace: kserve-ci-e2e-test resourceVersion: "25736" uid: 2205f917-dbd4-47d0-aa3b-d5a4030c8fba reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:56:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-0283f namespace: kserve-ci-e2e-test resourceVersion: "25717" uid: 3a1fbde1-3e63-423a-81ba-966d4db45fe3 kind: Event lastTimestamp: "2026-04-17T17:56:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:21Z" name: switch-graph-0283f.18a7368f21a189b5 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: bdcdefbf-f0f6-46f1-a045-7fda68ac29fc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:56:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-0283f namespace: kserve-ci-e2e-test resourceVersion: "25717" uid: 3a1fbde1-3e63-423a-81ba-966d4db45fe3 kind: Event lastTimestamp: "2026-04-17T17:56:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:56:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:56:21Z" name: switch-graph-0283f.18a7368f2210422a namespace: kserve-ci-e2e-test resourceVersion: "26044" uid: 02862992-f90d-49c5-b8b9-1e0138fbf33a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:34:11.251840Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-235f3-669999df9c-fr57z namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: 5e998f7a-f6a9-4c7b-86fc-62266b83449b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-235f3-669999df9c-fr57z to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:34:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:34:11Z" name: switch-graph-235f3-669999df9c-fr57z.18a7355ced160b25 namespace: kserve-ci-e2e-test resourceVersion: "14609" uid: f2e29b7e-e029-4f78-91d1-61de9b2ec88f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-235f3-669999df9c-fr57z namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 5e998f7a-f6a9-4c7b-86fc-62266b83449b kind: Event lastTimestamp: "2026-04-17T17:34:11Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:34:11Z" name: switch-graph-235f3-669999df9c-fr57z.18a7355d05f66f9c namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 85e2fd6b-c09b-4d5f-8994-fe12b0378ede reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-235f3} kind: Pod name: switch-graph-235f3-669999df9c-fr57z namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: 5e998f7a-f6a9-4c7b-86fc-62266b83449b kind: Event lastTimestamp: "2026-04-17T17:34:11Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:0589637b57a3315022b8ebf7fc1b0f9831e3a45e07a3fc2fab24137dd8f190f9" metadata: creationTimestamp: "2026-04-17T17:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:11Z" name: switch-graph-235f3-669999df9c-fr57z.18a7355d07420437 namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: 4fd2ef8a-2cb6-46c7-b6ba-757a67016369 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-235f3} kind: Pod name: switch-graph-235f3-669999df9c-fr57z namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: 5e998f7a-f6a9-4c7b-86fc-62266b83449b kind: Event lastTimestamp: "2026-04-17T17:34:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:0589637b57a3315022b8ebf7fc1b0f9831e3a45e07a3fc2fab24137dd8f190f9" in 2.798s (2.798s including waiting). Image size: 216233549 bytes.' metadata: creationTimestamp: "2026-04-17T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:14Z" name: switch-graph-235f3-669999df9c-fr57z.18a7355dae0bc67f namespace: kserve-ci-e2e-test resourceVersion: "14648" uid: 65022774-333f-4755-a2b0-892ac7d744a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-235f3} kind: Pod name: switch-graph-235f3-669999df9c-fr57z namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: 5e998f7a-f6a9-4c7b-86fc-62266b83449b kind: Event lastTimestamp: "2026-04-17T17:34:14Z" message: 'Created container: switch-graph-235f3' metadata: creationTimestamp: "2026-04-17T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:14Z" name: switch-graph-235f3-669999df9c-fr57z.18a7355db107972f namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: 1d7c8c98-84de-42a7-a730-0382b425f510 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-235f3} kind: Pod name: switch-graph-235f3-669999df9c-fr57z namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: 5e998f7a-f6a9-4c7b-86fc-62266b83449b kind: Event lastTimestamp: "2026-04-17T17:34:14Z" message: Started container switch-graph-235f3 metadata: creationTimestamp: "2026-04-17T17:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:14Z" name: switch-graph-235f3-669999df9c-fr57z.18a7355db189d5fd namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: b917d4d7-9233-49d5-af1d-07a46b97e63d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-235f3} kind: Pod name: switch-graph-235f3-669999df9c-fr57z namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: 5e998f7a-f6a9-4c7b-86fc-62266b83449b kind: Event lastTimestamp: "2026-04-17T17:34:21Z" message: Stopping container switch-graph-235f3 metadata: creationTimestamp: "2026-04-17T17:34:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:21Z" name: switch-graph-235f3-669999df9c-fr57z.18a7355f4da4e74e namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 0dec637e-b48f-49ca-9d45-f73d03166617 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:34:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-235f3} kind: Pod name: switch-graph-235f3-669999df9c-fr57z namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: 5e998f7a-f6a9-4c7b-86fc-62266b83449b kind: Event lastTimestamp: "2026-04-17T17:34:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:34:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:34:51Z" name: switch-graph-235f3-669999df9c-fr57z.18a7356067f20c47 namespace: kserve-ci-e2e-test resourceVersion: "15129" uid: a12d3512-2cf5-4b5c-9f3d-70b053d9da3b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-235f3-669999df9c namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: 698b0069-b6c5-466f-bab2-85b812c7fa0a kind: Event lastTimestamp: "2026-04-17T17:34:11Z" message: 'Created pod: switch-graph-235f3-669999df9c-fr57z' metadata: creationTimestamp: "2026-04-17T17:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:11Z" name: switch-graph-235f3-669999df9c.18a7355cecb6f4db namespace: kserve-ci-e2e-test resourceVersion: "14606" uid: eb61a0c4-de9d-4936-b56a-44897d350a04 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-235f3 namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: 9b45639a-30c1-457a-a8e4-b8151013fec1 kind: Event lastTimestamp: "2026-04-17T17:34:11Z" message: Scaled up replica set switch-graph-235f3-669999df9c from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:34:11Z" name: switch-graph-235f3.18a7355cebef93b3 namespace: kserve-ci-e2e-test resourceVersion: "14602" uid: 8f069113-e950-43ec-9ead-44fb3f22e46a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:34:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-235f3 namespace: kserve-ci-e2e-test resourceVersion: "14615" uid: e843d997-7b9c-4cf0-88c2-f4af190aedc5 kind: Event lastTimestamp: "2026-04-17T17:34:11Z" message: InferenceGraph [switch-graph-235f3] is Ready metadata: creationTimestamp: "2026-04-17T17:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:34:11Z" name: switch-graph-235f3.18a7355cef9b3c7d namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: 49315168-d015-4c31-93b2-7b6a50621980 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-17T17:35:21.675150Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d987b-6595588756-qw7ws namespace: kserve-ci-e2e-test resourceVersion: "15617" uid: fa2a2239-31a2-4a3b-8325-c38b645c4b7e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d987b-6595588756-qw7ws to ip-10-0-140-33.ec2.internal metadata: creationTimestamp: "2026-04-17T17:35:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-17T17:35:21Z" name: switch-graph-d987b-6595588756-qw7ws.18a7356d52a458b1 namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: cae5fefe-304f-4325-8875-6d95b236cc05 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b5c697c48-6v9ls source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:22Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d987b-6595588756-qw7ws namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: fa2a2239-31a2-4a3b-8325-c38b645c4b7e kind: Event lastTimestamp: "2026-04-17T17:35:22Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-17T17:35:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-17T17:35:22Z" name: switch-graph-d987b-6595588756-qw7ws.18a7356d6bf22d5f namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: 2221fe2b-af5c-4c79-9818-0888844730eb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d987b} kind: Pod name: switch-graph-d987b-6595588756-qw7ws namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: fa2a2239-31a2-4a3b-8325-c38b645c4b7e kind: Event lastTimestamp: "2026-04-17T17:35:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0589637b57a3315022b8ebf7fc1b0f9831e3a45e07a3fc2fab24137dd8f190f9" already present on machine metadata: creationTimestamp: "2026-04-17T17:35:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:22Z" name: switch-graph-d987b-6595588756-qw7ws.18a7356d6d3abd5e namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: 1dcb6f61-ae4c-4825-94ac-fd790a87b203 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d987b} kind: Pod name: switch-graph-d987b-6595588756-qw7ws namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: fa2a2239-31a2-4a3b-8325-c38b645c4b7e kind: Event lastTimestamp: "2026-04-17T17:35:22Z" message: 'Created container: switch-graph-d987b' metadata: creationTimestamp: "2026-04-17T17:35:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:22Z" name: switch-graph-d987b-6595588756-qw7ws.18a7356d70ad77a2 namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: 9eda058b-91d6-4334-8963-a3f6ac833a3f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d987b} kind: Pod name: switch-graph-d987b-6595588756-qw7ws namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: fa2a2239-31a2-4a3b-8325-c38b645c4b7e kind: Event lastTimestamp: "2026-04-17T17:35:22Z" message: Started container switch-graph-d987b metadata: creationTimestamp: "2026-04-17T17:35:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:35:22Z" name: switch-graph-d987b-6595588756-qw7ws.18a7356d711e86ec namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 1b5fbd5d-b052-4883-a27b-eddf87c08cf9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:43:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d987b} kind: Pod name: switch-graph-d987b-6595588756-qw7ws namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: fa2a2239-31a2-4a3b-8325-c38b645c4b7e kind: Event lastTimestamp: "2026-04-17T17:43:36Z" message: Stopping container switch-graph-d987b metadata: creationTimestamp: "2026-04-17T17:43:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:43:36Z" name: switch-graph-d987b-6595588756-qw7ws.18a735e0885a7e4a namespace: kserve-ci-e2e-test resourceVersion: "19122" uid: f1aabc98-9a06-4b64-a60b-fdaa71d4c0d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-17T17:43:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d987b} kind: Pod name: switch-graph-d987b-6595588756-qw7ws namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: fa2a2239-31a2-4a3b-8325-c38b645c4b7e kind: Event lastTimestamp: "2026-04-17T17:44:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-17T17:43:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-17T17:44:03Z" name: switch-graph-d987b-6595588756-qw7ws.18a735e0fc66a83b namespace: kserve-ci-e2e-test resourceVersion: "19456" uid: 6a792dc8-a2b2-4cc5-bf81-014129a0d10a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-33.ec2.internal source: component: kubelet host: ip-10-0-140-33.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d987b-6595588756 namespace: kserve-ci-e2e-test resourceVersion: "15613" uid: b508233d-8016-4416-8739-e1290b1c41c6 kind: Event lastTimestamp: "2026-04-17T17:35:21Z" message: 'Created pod: switch-graph-d987b-6595588756-qw7ws' metadata: creationTimestamp: "2026-04-17T17:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:21Z" name: switch-graph-d987b-6595588756.18a7356d524a6b27 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: 2887619b-fc63-484d-9193-249c1b31cdd2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d987b namespace: kserve-ci-e2e-test resourceVersion: "15611" uid: cfb42b7f-dc14-4de9-a76f-6113fb3aeefe kind: Event lastTimestamp: "2026-04-17T17:35:21Z" message: Scaled up replica set switch-graph-d987b-6595588756 from 0 to 1 metadata: creationTimestamp: "2026-04-17T17:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:21Z" name: switch-graph-d987b.18a7356d51b4e6ec namespace: kserve-ci-e2e-test resourceVersion: "15615" uid: aaecca14-eead-4787-b4c0-e189417fc1b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-17T17:35:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d987b namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: adde16db-1382-465c-a1b3-51d61eb91d21 kind: Event lastTimestamp: "2026-04-17T17:35:21Z" message: InferenceGraph [switch-graph-d987b] is Ready metadata: creationTimestamp: "2026-04-17T17:35:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-17T17:35:21Z" name: switch-graph-d987b.18a7356d5689abd1 namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 9671fe8a-35e3-4239-8341-bc9b99242c08 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:35:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d987b namespace: kserve-ci-e2e-test resourceVersion: "15612" uid: 75144e46-7021-4184-9219-f93f1a8530ab kind: Event lastTimestamp: "2026-04-17T17:35:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:51Z" name: switch-graph-d987b.18a73570d0910077 namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: 6146cf04-62d3-4751-a9d2-4636b8762cd4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-17T17:35:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d987b namespace: kserve-ci-e2e-test resourceVersion: "15612" uid: 75144e46-7021-4184-9219-f93f1a8530ab kind: Event lastTimestamp: "2026-04-17T17:35:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-17T17:35:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-17T17:35:51Z" name: switch-graph-d987b.18a73570d0f8fcab namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: d11a4e5a-8169-4d4b-a540-9a8b1867ee3b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33157"