--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:55:58.451773Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0a5df-77865c759f-kmm24 namespace: kserve-ci-e2e-test resourceVersion: "23048" uid: 9ddb2c1f-bbdf-48ee-a623-3791529de7f6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-0a5df-77865c759f-kmm24 to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:55:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:55:58Z" name: ensemble-graph-0a5df-77865c759f-kmm24.18b31b8a46fb2e4d namespace: kserve-ci-e2e-test resourceVersion: "23053" uid: 7e872c31-e891-4d9e-af35-72a74ff658fa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:58Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0a5df-77865c759f-kmm24 namespace: kserve-ci-e2e-test resourceVersion: "23050" uid: 9ddb2c1f-bbdf-48ee-a623-3791529de7f6 kind: Event lastTimestamp: "2026-05-26T11:55:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-0a5df-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:58Z" name: ensemble-graph-0a5df-77865c759f-kmm24.18b31b8a4fea241c namespace: kserve-ci-e2e-test resourceVersion: "23067" uid: cd6e6198-d218-48fe-be97-2b314c63292c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:59Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-0a5df-77865c759f-kmm24 namespace: kserve-ci-e2e-test resourceVersion: "23058" uid: 9ddb2c1f-bbdf-48ee-a623-3791529de7f6 kind: Event lastTimestamp: "2026-05-26T11:55:59Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:55:59Z" name: ensemble-graph-0a5df-77865c759f-kmm24.18b31b8a83fbd76c namespace: kserve-ci-e2e-test resourceVersion: "23077" uid: 24c0bf02-a39f-4e20-af0a-332094483dc4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0a5df} kind: Pod name: ensemble-graph-0a5df-77865c759f-kmm24 namespace: kserve-ci-e2e-test resourceVersion: "23050" uid: 9ddb2c1f-bbdf-48ee-a623-3791529de7f6 kind: Event lastTimestamp: "2026-05-26T11:55:59Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-26T11:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:59Z" name: ensemble-graph-0a5df-77865c759f-kmm24.18b31b8a85278d63 namespace: kserve-ci-e2e-test resourceVersion: "23080" uid: 227fc915-d40b-480e-9a06-68b16c1e489e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0a5df} kind: Pod name: ensemble-graph-0a5df-77865c759f-kmm24 namespace: kserve-ci-e2e-test resourceVersion: "23050" uid: 9ddb2c1f-bbdf-48ee-a623-3791529de7f6 kind: Event lastTimestamp: "2026-05-26T11:55:59Z" message: 'Created container: ensemble-graph-0a5df' metadata: creationTimestamp: "2026-05-26T11:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:59Z" name: ensemble-graph-0a5df-77865c759f-kmm24.18b31b8a87fa3cd2 namespace: kserve-ci-e2e-test resourceVersion: "23082" uid: a1e1ced5-4a07-4bac-8ed4-f3995f79cd2c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0a5df} kind: Pod name: ensemble-graph-0a5df-77865c759f-kmm24 namespace: kserve-ci-e2e-test resourceVersion: "23050" uid: 9ddb2c1f-bbdf-48ee-a623-3791529de7f6 kind: Event lastTimestamp: "2026-05-26T11:55:59Z" message: Started container ensemble-graph-0a5df metadata: creationTimestamp: "2026-05-26T11:55:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:59Z" name: ensemble-graph-0a5df-77865c759f-kmm24.18b31b8a88686059 namespace: kserve-ci-e2e-test resourceVersion: "23083" uid: 78f51c44-8267-4fa8-8e5a-7e33400a4c8e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0a5df} kind: Pod name: ensemble-graph-0a5df-77865c759f-kmm24 namespace: kserve-ci-e2e-test resourceVersion: "23050" uid: 9ddb2c1f-bbdf-48ee-a623-3791529de7f6 kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: Stopping container ensemble-graph-0a5df metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:13Z" name: ensemble-graph-0a5df-77865c759f-kmm24.18b31bfd75f483bc namespace: kserve-ci-e2e-test resourceVersion: "26868" uid: d1d52ff9-a4c0-4c5e-be2a-bd22d88e9810 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T12:04:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-0a5df} kind: Pod name: ensemble-graph-0a5df-77865c759f-kmm24 namespace: kserve-ci-e2e-test resourceVersion: "23050" uid: 9ddb2c1f-bbdf-48ee-a623-3791529de7f6 kind: Event lastTimestamp: "2026-05-26T12:04:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T12:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:40Z" name: ensemble-graph-0a5df-77865c759f-kmm24.18b31bfe1b273fc3 namespace: kserve-ci-e2e-test resourceVersion: "27214" uid: dc91f90b-0e27-4dbc-b938-01121280afcf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-0a5df-77865c759f namespace: kserve-ci-e2e-test resourceVersion: "23045" uid: 0b41fdb7-f39d-4a67-9efb-030afc2ad130 kind: Event lastTimestamp: "2026-05-26T11:55:58Z" message: 'Created pod: ensemble-graph-0a5df-77865c759f-kmm24' metadata: creationTimestamp: "2026-05-26T11:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:58Z" name: ensemble-graph-0a5df-77865c759f.18b31b8a468a51cd namespace: kserve-ci-e2e-test resourceVersion: "23049" uid: c57dbb60-f491-4795-be8f-b07b0ad3dd9f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-0a5df namespace: kserve-ci-e2e-test resourceVersion: "23043" uid: 8868647c-9d8b-48ca-9b2d-43f05cb09bcb kind: Event lastTimestamp: "2026-05-26T11:55:58Z" message: Scaled up replica set ensemble-graph-0a5df-77865c759f from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:58Z" name: ensemble-graph-0a5df.18b31b8a459d7e66 namespace: kserve-ci-e2e-test resourceVersion: "23046" uid: 73c14311-cb57-4718-ac43-811775dcfa7d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:58Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-0a5df namespace: kserve-ci-e2e-test resourceVersion: "23062" uid: 8bbc5923-b9c0-400e-90a4-0bc465d9ee01 kind: Event lastTimestamp: "2026-05-26T11:55:58Z" message: InferenceGraph [ensemble-graph-0a5df] is Ready metadata: creationTimestamp: "2026-05-26T11:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:58Z" name: ensemble-graph-0a5df.18b31b8a4bc0e8b4 namespace: kserve-ci-e2e-test resourceVersion: "23064" uid: 15036e4c-ff3a-47e0-b12f-d560a99714a8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-0a5df namespace: kserve-ci-e2e-test resourceVersion: "23044" uid: 54abd35a-07a6-48bf-89a2-3482d547f327 kind: Event lastTimestamp: "2026-05-26T11:56:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:13Z" name: ensemble-graph-0a5df.18b31b8dc40917fb namespace: kserve-ci-e2e-test resourceVersion: "23189" uid: 55ed8927-9ff6-40dd-9e31-b297257b5180 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-0a5df namespace: kserve-ci-e2e-test resourceVersion: "23044" uid: 54abd35a-07a6-48bf-89a2-3482d547f327 kind: Event lastTimestamp: "2026-05-26T11:56:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:13Z" name: ensemble-graph-0a5df.18b31b8dc46f9594 namespace: kserve-ci-e2e-test resourceVersion: "23191" uid: e013a85c-68f3-4071-b945-0194559f3e81 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-0a5df namespace: kserve-ci-e2e-test resourceVersion: "23190" uid: 54abd35a-07a6-48bf-89a2-3482d547f327 kind: Event lastTimestamp: "2026-05-26T11:56:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:28Z" name: ensemble-graph-0a5df.18b31b91431b4749 namespace: kserve-ci-e2e-test resourceVersion: "23354" uid: 574be871-ee82-49c3-9266-1bb1693d20b3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-0a5df namespace: kserve-ci-e2e-test resourceVersion: "23190" uid: 54abd35a-07a6-48bf-89a2-3482d547f327 kind: Event lastTimestamp: "2026-05-26T11:56:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:28Z" name: ensemble-graph-0a5df.18b31b9143888f83 namespace: kserve-ci-e2e-test resourceVersion: "23357" uid: dfc55c2e-b0a7-44a3-b015-ba121dcb4022 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:54:48.130370Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5380a-676cb95c4b-phg74 namespace: kserve-ci-e2e-test resourceVersion: "22074" uid: 7d1acaa3-e946-40d3-9937-ef994cf95b0a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-5380a-676cb95c4b-phg74 to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:54:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:54:48Z" name: ensemble-graph-5380a-676cb95c4b-phg74.18b31b79e77fc001 namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: c6858dfd-ed02-4402-a1c7-7163f111397e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:48Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5380a-676cb95c4b-phg74 namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: 7d1acaa3-e946-40d3-9937-ef994cf95b0a kind: Event lastTimestamp: "2026-05-26T11:54:48Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:54:48Z" name: ensemble-graph-5380a-676cb95c4b-phg74.18b31b7a01871ffd namespace: kserve-ci-e2e-test resourceVersion: "22097" uid: cece5b0d-ae0f-435b-bcc5-e6c26482e5fb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5380a} kind: Pod name: ensemble-graph-5380a-676cb95c4b-phg74 namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: 7d1acaa3-e946-40d3-9937-ef994cf95b0a kind: Event lastTimestamp: "2026-05-26T11:54:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-26T11:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:48Z" name: ensemble-graph-5380a-676cb95c4b-phg74.18b31b7a02d6eb0c namespace: kserve-ci-e2e-test resourceVersion: "22099" uid: ec982b94-53b1-4bad-baca-4ee95768857a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5380a} kind: Pod name: ensemble-graph-5380a-676cb95c4b-phg74 namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: 7d1acaa3-e946-40d3-9937-ef994cf95b0a kind: Event lastTimestamp: "2026-05-26T11:54:48Z" message: 'Created container: ensemble-graph-5380a' metadata: creationTimestamp: "2026-05-26T11:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:48Z" name: ensemble-graph-5380a-676cb95c4b-phg74.18b31b7a05cbc2aa namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: 6f919c75-6fdf-48c5-9dd0-dc28f5a515cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5380a} kind: Pod name: ensemble-graph-5380a-676cb95c4b-phg74 namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: 7d1acaa3-e946-40d3-9937-ef994cf95b0a kind: Event lastTimestamp: "2026-05-26T11:54:48Z" message: Started container ensemble-graph-5380a metadata: creationTimestamp: "2026-05-26T11:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:48Z" name: ensemble-graph-5380a-676cb95c4b-phg74.18b31b7a0652dd00 namespace: kserve-ci-e2e-test resourceVersion: "22103" uid: 8cba2735-cd6c-4188-b385-96bb620ba097 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5380a} kind: Pod name: ensemble-graph-5380a-676cb95c4b-phg74 namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: 7d1acaa3-e946-40d3-9937-ef994cf95b0a kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: Stopping container ensemble-graph-5380a metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:58Z" name: ensemble-graph-5380a-676cb95c4b-phg74.18b31b7c42357aae namespace: kserve-ci-e2e-test resourceVersion: "22196" uid: 6986bb74-00b0-45b8-b743-ea74b2aa0209 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5380a} kind: Pod name: ensemble-graph-5380a-676cb95c4b-phg74 namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: 7d1acaa3-e946-40d3-9937-ef994cf95b0a kind: Event lastTimestamp: "2026-05-26T11:55:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:24Z" name: ensemble-graph-5380a-676cb95c4b-phg74.18b31b7c99603dc2 namespace: kserve-ci-e2e-test resourceVersion: "22588" uid: 8e5eb111-b927-4045-910e-516b123f1c3b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-5380a-676cb95c4b namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: 4f38df14-a014-49a4-971a-f7cd3ea9c68e kind: Event lastTimestamp: "2026-05-26T11:54:48Z" message: 'Created pod: ensemble-graph-5380a-676cb95c4b-phg74' metadata: creationTimestamp: "2026-05-26T11:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:48Z" name: ensemble-graph-5380a-676cb95c4b.18b31b79e6ec741e namespace: kserve-ci-e2e-test resourceVersion: "22075" uid: f96a16ff-c5cd-4d24-9249-5fb06a05d26d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-5380a namespace: kserve-ci-e2e-test resourceVersion: "22068" uid: c9af1cb1-2812-4f30-a11e-973a4a8d3186 kind: Event lastTimestamp: "2026-05-26T11:54:48Z" message: Scaled up replica set ensemble-graph-5380a-676cb95c4b from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:48Z" name: ensemble-graph-5380a.18b31b79e623bf3e namespace: kserve-ci-e2e-test resourceVersion: "22072" uid: 1ea4f3de-b0c8-4ef2-b2c4-10a67abab265 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-5380a namespace: kserve-ci-e2e-test resourceVersion: "22087" uid: bb57e1ce-7936-49ee-aee2-134d14d83ddc kind: Event lastTimestamp: "2026-05-26T11:54:48Z" message: InferenceGraph [ensemble-graph-5380a] is Ready metadata: creationTimestamp: "2026-05-26T11:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:48Z" name: ensemble-graph-5380a.18b31b79eb5e7945 namespace: kserve-ci-e2e-test resourceVersion: "22089" uid: 48ff25e3-8b06-40fa-afba-da8a6e55f2f7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-5380a namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: bb57e1ce-7936-49ee-aee2-134d14d83ddc kind: Event lastTimestamp: "2026-05-26T11:54:48Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-5380a": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-5380a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:48Z" name: ensemble-graph-5380a.18b31b79ed633d9b namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: dafae69d-b2ec-4737-ab50-f4db49698252 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-5380a namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: bb57e1ce-7936-49ee-aee2-134d14d83ddc kind: Event lastTimestamp: "2026-05-26T11:54:48Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-5380a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:48Z" name: ensemble-graph-5380a.18b31b79ed638c33 namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: 13191b64-7ff9-45e0-9526-13c7b3029d0c reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:54:58.513452Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0a5df-predictor-c47858789-v9hph namespace: kserve-ci-e2e-test resourceVersion: "22273" uid: 68b2c4c5-b1c4-4441-b57d-8dac155f8799 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0a5df-predictor-c47858789-v9hph to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:54:58Z" name: error-404-isvc-0a5df-predictor-c47858789-v9hph.18b31b7c526103e7 namespace: kserve-ci-e2e-test resourceVersion: "22280" uid: 7d81b705-a8e8-40fa-91a7-ea6c688a0702 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0a5df-predictor-c47858789-v9hph namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 68b2c4c5-b1c4-4441-b57d-8dac155f8799 kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-0a5df-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:58Z" name: error-404-isvc-0a5df-predictor-c47858789-v9hph.18b31b7c5dedc2f4 namespace: kserve-ci-e2e-test resourceVersion: "22286" uid: 51df1c45-ee55-452d-a676-d3aaa4997bd4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0a5df-predictor-c47858789-v9hph namespace: kserve-ci-e2e-test resourceVersion: "22283" uid: 68b2c4c5-b1c4-4441-b57d-8dac155f8799 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:54:59Z" name: error-404-isvc-0a5df-predictor-c47858789-v9hph.18b31b7c8f894753 namespace: kserve-ci-e2e-test resourceVersion: "22306" uid: a23981d5-f828-4260-b14a-689e8002bbcf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a5df-predictor-c47858789-v9hph namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 68b2c4c5-b1c4-4441-b57d-8dac155f8799 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:59Z" name: error-404-isvc-0a5df-predictor-c47858789-v9hph.18b31b7c90dff0eb namespace: kserve-ci-e2e-test resourceVersion: "22308" uid: 8c29d173-f51d-40b5-90ca-3a8ba359e79e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a5df-predictor-c47858789-v9hph namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 68b2c4c5-b1c4-4441-b57d-8dac155f8799 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:59Z" name: error-404-isvc-0a5df-predictor-c47858789-v9hph.18b31b7c942cf6fc namespace: kserve-ci-e2e-test resourceVersion: "22311" uid: d1f3d903-63c5-4f26-9b14-2f3a3e87429e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a5df-predictor-c47858789-v9hph namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 68b2c4c5-b1c4-4441-b57d-8dac155f8799 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:59Z" name: error-404-isvc-0a5df-predictor-c47858789-v9hph.18b31b7c94a7d7a2 namespace: kserve-ci-e2e-test resourceVersion: "22312" uid: 4b0c7683-5e52-4a42-a430-4436d3982cf3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a5df-predictor-c47858789-v9hph namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 68b2c4c5-b1c4-4441-b57d-8dac155f8799 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:59Z" name: error-404-isvc-0a5df-predictor-c47858789-v9hph.18b31b7c94b4b3db namespace: kserve-ci-e2e-test resourceVersion: "22313" uid: da7cf1b8-ec90-49ab-9d68-db1ecf57c48e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a5df-predictor-c47858789-v9hph namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 68b2c4c5-b1c4-4441-b57d-8dac155f8799 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:59Z" name: error-404-isvc-0a5df-predictor-c47858789-v9hph.18b31b7c975c61a8 namespace: kserve-ci-e2e-test resourceVersion: "22316" uid: 43a8d21e-981a-4fe0-bfcb-066352c22d07 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a5df-predictor-c47858789-v9hph namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 68b2c4c5-b1c4-4441-b57d-8dac155f8799 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:59Z" name: error-404-isvc-0a5df-predictor-c47858789-v9hph.18b31b7c97d03bdc namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: 92a46408-a42e-445e-99d2-22588fa7940d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a5df-predictor-c47858789-v9hph namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 68b2c4c5-b1c4-4441-b57d-8dac155f8799 kind: Event lastTimestamp: "2026-05-26T12:04:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:15Z" name: error-404-isvc-0a5df-predictor-c47858789-v9hph.18b31b7c9af3f364 namespace: kserve-ci-e2e-test resourceVersion: "27009" uid: 3c7b8d51-5786-4a60-b6b5-f69ca8fbf6a7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0a5df-predictor-c47858789-v9hph namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 68b2c4c5-b1c4-4441-b57d-8dac155f8799 kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:13Z" name: error-404-isvc-0a5df-predictor-c47858789-v9hph.18b31bfd8453007b namespace: kserve-ci-e2e-test resourceVersion: "26921" uid: aaedeb59-9cfd-4751-b813-045e13391521 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a5df-predictor-c47858789-v9hph namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 68b2c4c5-b1c4-4441-b57d-8dac155f8799 kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:13Z" name: error-404-isvc-0a5df-predictor-c47858789-v9hph.18b31bfd84542659 namespace: kserve-ci-e2e-test resourceVersion: "26923" uid: 8b01e6f8-ade3-4e05-bfd0-2a03768aafda reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0a5df-predictor-c47858789-v9hph namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: 68b2c4c5-b1c4-4441-b57d-8dac155f8799 kind: Event lastTimestamp: "2026-05-26T12:04:15Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T12:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:15Z" name: error-404-isvc-0a5df-predictor-c47858789-v9hph.18b31bfe0f4f5213 namespace: kserve-ci-e2e-test resourceVersion: "27008" uid: 6e01df87-95d5-499f-b192-2af31072c80f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0a5df-predictor-c47858789 namespace: kserve-ci-e2e-test resourceVersion: "22266" uid: ddb83ec4-ec28-48c9-82c5-711a7d6749eb kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: 'Created pod: error-404-isvc-0a5df-predictor-c47858789-v9hph' metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:58Z" name: error-404-isvc-0a5df-predictor-c47858789.18b31b7c514ef5c9 namespace: kserve-ci-e2e-test resourceVersion: "22274" uid: ecc64696-571b-4129-8924-ce7218affe0a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0a5df-predictor namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: d4e33b85-01c0-49ea-b63a-4c21d7d31983 kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: Scaled up replica set error-404-isvc-0a5df-predictor-c47858789 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:58Z" name: error-404-isvc-0a5df-predictor.18b31b7c4f1e1834 namespace: kserve-ci-e2e-test resourceVersion: "22268" uid: de8df30f-6e4c-4335-9ce0-e4b89b2a9f69 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:55:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0a5df-predictor namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: 592a1008-5ef2-4f48-8296-7eedf38236f6 kind: Event lastTimestamp: "2026-05-26T11:56:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:13Z" name: error-404-isvc-0a5df-predictor.18b31b7fcd967627 namespace: kserve-ci-e2e-test resourceVersion: "23192" uid: 63bcb043-651e-4bbe-8e9b-4d799617f4fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:55:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0a5df-predictor namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: 592a1008-5ef2-4f48-8296-7eedf38236f6 kind: Event lastTimestamp: "2026-05-26T11:56:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:13Z" name: error-404-isvc-0a5df-predictor.18b31b7fcdfb313b namespace: kserve-ci-e2e-test resourceVersion: "23193" uid: 04e3479f-d30d-4ed0-bf4a-390513d7be7d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0a5df namespace: kserve-ci-e2e-test resourceVersion: "22250" uid: 7e55196d-02c9-405b-a57e-c851c5fcf0cb kind: Event lastTimestamp: "2026-05-26T11:55:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:45Z" name: error-404-isvc-0a5df.18b31b7c4ef34b9f namespace: kserve-ci-e2e-test resourceVersion: "22921" uid: 843d7fd7-652b-4c30-b323-7ee2fcf1b240 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0a5df namespace: kserve-ci-e2e-test resourceVersion: "22250" uid: 7e55196d-02c9-405b-a57e-c851c5fcf0cb kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: 'Failed to update status for InferenceService "error-404-isvc-0a5df": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0a5df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:58Z" name: error-404-isvc-0a5df.18b31b7c4f61935a namespace: kserve-ci-e2e-test resourceVersion: "22270" uid: b784cf9f-51d9-4c84-81da-f3727a5808ba reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0a5df namespace: kserve-ci-e2e-test resourceVersion: "22250" uid: 7e55196d-02c9-405b-a57e-c851c5fcf0cb kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0a5df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:58Z" name: error-404-isvc-0a5df.18b31b7c4f61e636 namespace: kserve-ci-e2e-test resourceVersion: "22271" uid: c26f63c5-d05d-46b3-82e0-b24102707b6f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0a5df namespace: kserve-ci-e2e-test resourceVersion: "22919" uid: 7e55196d-02c9-405b-a57e-c851c5fcf0cb kind: Event lastTimestamp: "2026-05-26T11:55:45Z" message: InferenceService [error-404-isvc-0a5df] is Ready metadata: creationTimestamp: "2026-05-26T11:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:45Z" name: error-404-isvc-0a5df.18b31b8755ca7c8b namespace: kserve-ci-e2e-test resourceVersion: "22920" uid: 98a23ba7-9ea3-4b53-bd9c-9cf9c4519d56 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T12:04:43.048539Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9 namespace: kserve-ci-e2e-test resourceVersion: "27323" uid: 7f7d4ea2-96ee-47b8-87f0-2cdd2e684186 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-152bc-predictor-6b9cb766b7-4krt9 to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9.18b31c046b629e57 namespace: kserve-ci-e2e-test resourceVersion: "27327" uid: 73899af0-8c21-4e12-9b8e-752420123744 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9 namespace: kserve-ci-e2e-test resourceVersion: "27333" uid: 7f7d4ea2-96ee-47b8-87f0-2cdd2e684186 kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9.18b31c04853aa48e namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: 08bdc137-3d25-4310-9056-5ee132a7b551 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f7d4ea2-96ee-47b8-87f0-2cdd2e684186 kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9.18b31c04877d9d39 namespace: kserve-ci-e2e-test resourceVersion: "27351" uid: 9d6d5c3c-0729-4373-ae94-1408c2670c5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f7d4ea2-96ee-47b8-87f0-2cdd2e684186 kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9.18b31c048a41db7d namespace: kserve-ci-e2e-test resourceVersion: "27352" uid: 6a0c7df1-149f-4d1d-b1f3-3778719a7269 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f7d4ea2-96ee-47b8-87f0-2cdd2e684186 kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9.18b31c048aba6d3e namespace: kserve-ci-e2e-test resourceVersion: "27353" uid: 23f32408-266d-4331-8a74-c6e772e1aa33 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f7d4ea2-96ee-47b8-87f0-2cdd2e684186 kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9.18b31c048ac6f001 namespace: kserve-ci-e2e-test resourceVersion: "27354" uid: 8b98ccea-2dab-4d29-a3b5-fbb44c505abe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f7d4ea2-96ee-47b8-87f0-2cdd2e684186 kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9.18b31c048e0cc98b namespace: kserve-ci-e2e-test resourceVersion: "27356" uid: e49ed51f-f661-4419-a517-c3e82383850b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f7d4ea2-96ee-47b8-87f0-2cdd2e684186 kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9.18b31c048e8225e3 namespace: kserve-ci-e2e-test resourceVersion: "27357" uid: c8a955d9-48c4-4160-905a-826df1cb0a41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T12:04:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f7d4ea2-96ee-47b8-87f0-2cdd2e684186 kind: Event lastTimestamp: "2026-05-26T12:05:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T12:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:21Z" name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9.18b31c04f9e242ee namespace: kserve-ci-e2e-test resourceVersion: "27743" uid: fb30bae2-35bc-462d-bb99-43ab12c97585 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f7d4ea2-96ee-47b8-87f0-2cdd2e684186 kind: Event lastTimestamp: "2026-05-26T12:22:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T12:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:22:02Z" name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9.18b31cf67a53c3dc namespace: kserve-ci-e2e-test resourceVersion: "34996" uid: fab7114d-8532-4db2-bb9f-7765dca45285 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f7d4ea2-96ee-47b8-87f0-2cdd2e684186 kind: Event lastTimestamp: "2026-05-26T12:22:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:22:02Z" name: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9.18b31cf67a546d6d namespace: kserve-ci-e2e-test resourceVersion: "34997" uid: d2b16f9e-6a5c-4c3c-80df-01d079dc7012 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-152bc-predictor-6b9cb766b7 namespace: kserve-ci-e2e-test resourceVersion: "27316" uid: 9fce4ca0-a090-4d49-8f54-f176c28f5fff kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: 'Created pod: error-404-isvc-152bc-predictor-6b9cb766b7-4krt9' metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-152bc-predictor-6b9cb766b7.18b31c046ad8f102 namespace: kserve-ci-e2e-test resourceVersion: "27324" uid: 98f635a6-4585-4ed9-bc56-bb5e1d4b51fe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-152bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "27314" uid: 6c06b1ed-4aa7-4936-ae4d-6bf95183d834 kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: Scaled up replica set error-404-isvc-152bc-predictor-6b9cb766b7 from 0 to 1 metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-152bc-predictor.18b31c0469210bcb namespace: kserve-ci-e2e-test resourceVersion: "27318" uid: efc25cb9-9c01-459b-b25b-b4331cc55c8e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T12:04:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-152bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "27315" uid: acdb980c-e68a-4300-b194-321d598f3b76 kind: Event lastTimestamp: "2026-05-26T12:05:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:58Z" name: error-404-isvc-152bc-predictor.18b31c07e8358226 namespace: kserve-ci-e2e-test resourceVersion: "28252" uid: da6afbef-c3c2-4bea-9486-feacf5c7b189 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T12:04:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-152bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "27315" uid: acdb980c-e68a-4300-b194-321d598f3b76 kind: Event lastTimestamp: "2026-05-26T12:05:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:58Z" name: error-404-isvc-152bc-predictor.18b31c07e8a6eabb namespace: kserve-ci-e2e-test resourceVersion: "28253" uid: 13a23b2c-5844-4c50-9add-953ca796dbe1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:05:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-152bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "27494" uid: acdb980c-e68a-4300-b194-321d598f3b76 kind: Event lastTimestamp: "2026-05-26T12:05:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:43Z" name: error-404-isvc-152bc-predictor.18b31c0ee6494213 namespace: kserve-ci-e2e-test resourceVersion: "28111" uid: 085778c2-e2fc-4cd5-b4dd-089d3c329fd9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:05:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-152bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "27494" uid: acdb980c-e68a-4300-b194-321d598f3b76 kind: Event lastTimestamp: "2026-05-26T12:05:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:43Z" name: error-404-isvc-152bc-predictor.18b31c0ee6a70305 namespace: kserve-ci-e2e-test resourceVersion: "28112" uid: da15df09-8e8c-4e96-a335-0f148f8fe04a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-152bc namespace: kserve-ci-e2e-test resourceVersion: "27304" uid: 27567a91-e2c3-40a1-abe6-5ed107700c69 kind: Event lastTimestamp: "2026-05-26T12:05:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:31Z" name: error-404-isvc-152bc.18b31c0468f16452 namespace: kserve-ci-e2e-test resourceVersion: "27991" uid: 1d229c1c-f16a-4c22-be3b-e5747e150ba0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-152bc namespace: kserve-ci-e2e-test resourceVersion: "27304" uid: 27567a91-e2c3-40a1-abe6-5ed107700c69 kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: 'Failed to update status for InferenceService "error-404-isvc-152bc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-152bc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-152bc.18b31c04694bc66f namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: d46c807f-f710-40e8-9506-ffef6c1e1a37 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-152bc namespace: kserve-ci-e2e-test resourceVersion: "27304" uid: 27567a91-e2c3-40a1-abe6-5ed107700c69 kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-152bc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-152bc.18b31c04694c1a26 namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: 384f41fb-2850-46c6-86f5-a0998dd47640 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-152bc namespace: kserve-ci-e2e-test resourceVersion: "27989" uid: 27567a91-e2c3-40a1-abe6-5ed107700c69 kind: Event lastTimestamp: "2026-05-26T12:05:31Z" message: InferenceService [error-404-isvc-152bc] is Ready metadata: creationTimestamp: "2026-05-26T12:05:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:31Z" name: error-404-isvc-152bc.18b31c0fb4fc61c3 namespace: kserve-ci-e2e-test resourceVersion: "27990" uid: 377721ed-fa55-4534-80dc-50e581f9dba5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:43:03.056850Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-378f2-predictor-d6b46746f-8chzn to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:43:03Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31ad5bdd79547 namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 12ef3236-3855-4983-9964-0cf193a68c34 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: "2026-05-26T11:43:03Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:43:03Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31ad5d772996f namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: b51870b7-bd9d-4d91-ab02-d26d3e432f12 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: "2026-05-26T11:43:03Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:03Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31ad5da1db5f3 namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 3fc4b3d2-6ace-42d3-882a-227a9d5b6e48 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: "2026-05-26T11:43:15Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" in 11.878s (11.878s including waiting). Image size: 1334724716 bytes.' metadata: creationTimestamp: "2026-05-26T11:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:15Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31ad89e24e714 namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 75c4d584-8639-4b6a-aacf-3e9d4ad488c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: "2026-05-26T11:43:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:15Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31ad8a20d3208 namespace: kserve-ci-e2e-test resourceVersion: "16143" uid: 98c5b6d7-7721-4958-bb76-46039e04c80b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: "2026-05-26T11:43:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:15Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31ad8a294199f namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: 6114d935-da70-4d97-938c-bbff48253cf0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: "2026-05-26T11:43:15Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-26T11:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:15Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31ad8a2a5a373 namespace: kserve-ci-e2e-test resourceVersion: "16145" uid: 1c4bd83e-a67e-4e4e-93b8-293cc1fd1dbb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: "2026-05-26T11:43:17Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.499s (2.499s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-26T11:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:17Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31ad937a4d1c7 namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: 63797f8b-dcf4-4122-a4ee-1054a1f137f9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: "2026-05-26T11:43:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:18Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31ad93b4785af namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 2aba192b-d5b4-45f4-b822-5beaa8aabad4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: "2026-05-26T11:43:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:18Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31ad93bbedced namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: 09b18e5d-b3c1-4019-820c-44346d6df488 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T11:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: "2026-05-26T11:44:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:35Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31ad99c341fd2 namespace: kserve-ci-e2e-test resourceVersion: "17087" uid: 43874e16-0448-4b1e-b945-2cdf7274c2f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31aeab3c894bf namespace: kserve-ci-e2e-test resourceVersion: "16991" uid: 2f205345-8d44-49de-9795-9442d80382ac reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31aeab3c91e5b namespace: kserve-ci-e2e-test resourceVersion: "16993" uid: fad8af32-8e31-486e-baa6-d038aea8f0ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-378f2-predictor-d6b46746f-8chzn namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 8277b22b-acea-47e9-932e-37813db5a360 kind: Event lastTimestamp: "2026-05-26T11:44:35Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:44:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:35Z" name: error-404-isvc-378f2-predictor-d6b46746f-8chzn.18b31aeb4e542c21 namespace: kserve-ci-e2e-test resourceVersion: "17086" uid: 47ff6179-0f83-4fcf-b0ad-c2c77910900d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-378f2-predictor-d6b46746f namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: a4f29155-7755-4ed6-8bac-a620f016c6ac kind: Event lastTimestamp: "2026-05-26T11:43:03Z" message: 'Created pod: error-404-isvc-378f2-predictor-d6b46746f-8chzn' metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:03Z" name: error-404-isvc-378f2-predictor-d6b46746f.18b31ad5bd183609 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 6f5a4bbf-f52a-49b4-8248-fb15fd5c48db reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-378f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: d2a5756e-da7e-4adc-b628-4384c8498ec2 kind: Event lastTimestamp: "2026-05-26T11:43:02Z" message: Scaled up replica set error-404-isvc-378f2-predictor-d6b46746f from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:02Z" name: error-404-isvc-378f2-predictor.18b31ad5b7b66120 namespace: kserve-ci-e2e-test resourceVersion: "15918" uid: d11a9a1a-7278-47e9-97a0-4652e75eeb9a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:43:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-378f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: 94fb7818-291a-4377-89a6-7dbd0eda20b4 kind: Event lastTimestamp: "2026-05-26T11:43:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:47Z" name: error-404-isvc-378f2-predictor.18b31ad9363878c6 namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: de3d9ca5-2788-4b24-8040-ad1e9d95633e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:43:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-378f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: 94fb7818-291a-4377-89a6-7dbd0eda20b4 kind: Event lastTimestamp: "2026-05-26T11:43:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:47Z" name: error-404-isvc-378f2-predictor.18b31ad936a8a675 namespace: kserve-ci-e2e-test resourceVersion: "16478" uid: 152c09f2-71fa-4d87-865d-2a1f3997c923 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:44:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-378f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 94fb7818-291a-4377-89a6-7dbd0eda20b4 kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-378f2-predictor.18b31ae3b2fb9538 namespace: kserve-ci-e2e-test resourceVersion: "16974" uid: 3a01d639-5ec1-4b45-a6ed-7cd4f7bb7074 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:44:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-378f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 94fb7818-291a-4377-89a6-7dbd0eda20b4 kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:44:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-378f2-predictor.18b31ae3b3683065 namespace: kserve-ci-e2e-test resourceVersion: "16980" uid: e9d82a9e-c240-4d52-9d0e-881573e85d17 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:43:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-378f2 namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: 50bcfbf3-744c-4227-b6a7-10696bf301c5 kind: Event lastTimestamp: "2026-05-26T11:44:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:05Z" name: error-404-isvc-378f2.18b31ad5b7771c6a namespace: kserve-ci-e2e-test resourceVersion: "16640" uid: 03ee4f10-a604-406b-a288-f41a02a28fc9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-378f2 namespace: kserve-ci-e2e-test resourceVersion: "16638" uid: 50bcfbf3-744c-4227-b6a7-10696bf301c5 kind: Event lastTimestamp: "2026-05-26T11:44:05Z" message: InferenceService [error-404-isvc-378f2] is Ready metadata: creationTimestamp: "2026-05-26T11:44:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:05Z" name: error-404-isvc-378f2.18b31ae456f67fd0 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 0f00c43b-d096-408f-8f64-112f5b9a9702 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:55:28.290861Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw namespace: kserve-ci-e2e-test resourceVersion: "22717" uid: b21b40a6-9db5-4117-8ad1-b1f087ed9ecc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3c458-predictor-64658f5dcf-rcnxw to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:55:28Z" name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw.18b31b8341403157 namespace: kserve-ci-e2e-test resourceVersion: "22721" uid: b86f7244-ce75-4ec6-a884-f8860839b605 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw namespace: kserve-ci-e2e-test resourceVersion: "22719" uid: b21b40a6-9db5-4117-8ad1-b1f087ed9ecc kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-3c458-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:28Z" name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw.18b31b834bb8d290 namespace: kserve-ci-e2e-test resourceVersion: "22735" uid: 4d542c90-a8fd-4d33-bace-f5c832a65156 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw namespace: kserve-ci-e2e-test resourceVersion: "22727" uid: b21b40a6-9db5-4117-8ad1-b1f087ed9ecc kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:55:29Z" name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw.18b31b837f46af4e namespace: kserve-ci-e2e-test resourceVersion: "22758" uid: 1b699dc1-76bd-4f34-968e-4b4ef2a73402 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw namespace: kserve-ci-e2e-test resourceVersion: "22719" uid: b21b40a6-9db5-4117-8ad1-b1f087ed9ecc kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:29Z" name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw.18b31b83809f7896 namespace: kserve-ci-e2e-test resourceVersion: "22761" uid: 92d212e1-91a4-417a-b419-5edc5e005071 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw namespace: kserve-ci-e2e-test resourceVersion: "22719" uid: b21b40a6-9db5-4117-8ad1-b1f087ed9ecc kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:29Z" name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw.18b31b8383784cb0 namespace: kserve-ci-e2e-test resourceVersion: "22764" uid: be7e4617-b1a6-4500-9b10-cd95e573ee23 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw namespace: kserve-ci-e2e-test resourceVersion: "22719" uid: b21b40a6-9db5-4117-8ad1-b1f087ed9ecc kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:29Z" name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw.18b31b8383f09258 namespace: kserve-ci-e2e-test resourceVersion: "22765" uid: cb66206c-d77d-428a-8104-bfc639683729 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw namespace: kserve-ci-e2e-test resourceVersion: "22719" uid: b21b40a6-9db5-4117-8ad1-b1f087ed9ecc kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:29Z" name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw.18b31b8383fd10cf namespace: kserve-ci-e2e-test resourceVersion: "22766" uid: 9a90e10d-6914-4f57-890a-5dfa81477d86 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw namespace: kserve-ci-e2e-test resourceVersion: "22719" uid: b21b40a6-9db5-4117-8ad1-b1f087ed9ecc kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:29Z" name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw.18b31b8386f2509f namespace: kserve-ci-e2e-test resourceVersion: "22767" uid: 4544cf49-f615-4728-b671-d4910a8205c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw namespace: kserve-ci-e2e-test resourceVersion: "22719" uid: b21b40a6-9db5-4117-8ad1-b1f087ed9ecc kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:29Z" name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw.18b31b83876aa527 namespace: kserve-ci-e2e-test resourceVersion: "22768" uid: d6e2de86-1396-4822-92ce-4d6c5aacef87 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:55:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw namespace: kserve-ci-e2e-test resourceVersion: "22719" uid: b21b40a6-9db5-4117-8ad1-b1f087ed9ecc kind: Event lastTimestamp: "2026-05-26T11:56:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:56:06Z" name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw.18b31b83d89dc768 namespace: kserve-ci-e2e-test resourceVersion: "23138" uid: 1e9470c9-34b5-452e-90b9-a6526e9b82f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw namespace: kserve-ci-e2e-test resourceVersion: "22719" uid: b21b40a6-9db5-4117-8ad1-b1f087ed9ecc kind: Event lastTimestamp: "2026-05-26T12:04:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T12:04:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:42Z" name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw.18b31c04601ba332 namespace: kserve-ci-e2e-test resourceVersion: "27265" uid: 16c39ae5-21d6-449f-9933-2da28f8a5563 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw namespace: kserve-ci-e2e-test resourceVersion: "22719" uid: b21b40a6-9db5-4117-8ad1-b1f087ed9ecc kind: Event lastTimestamp: "2026-05-26T12:04:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:04:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:42Z" name: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw.18b31c04601c1b79 namespace: kserve-ci-e2e-test resourceVersion: "27268" uid: ec06ede7-9fc7-4cc6-9396-c637ed20143b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3c458-predictor-64658f5dcf namespace: kserve-ci-e2e-test resourceVersion: "22709" uid: 5890014b-6cd5-4309-9f36-ef719beae930 kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: 'Created pod: error-404-isvc-3c458-predictor-64658f5dcf-rcnxw' metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:28Z" name: error-404-isvc-3c458-predictor-64658f5dcf.18b31b8340c26cb5 namespace: kserve-ci-e2e-test resourceVersion: "22718" uid: db6e1e04-aa57-475e-8253-f56b49c37de7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3c458-predictor namespace: kserve-ci-e2e-test resourceVersion: "22706" uid: f56df74d-374a-4185-9f96-f1fede8b15dd kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: Scaled up replica set error-404-isvc-3c458-predictor-64658f5dcf from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:28Z" name: error-404-isvc-3c458-predictor.18b31b833f469a9e namespace: kserve-ci-e2e-test resourceVersion: "22711" uid: 53df86d4-8e74-4924-8c4d-c0502c6b8ce0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:55:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c458-predictor namespace: kserve-ci-e2e-test resourceVersion: "22708" uid: 808f594a-c5f1-45c4-8b08-0308a8e9aa00 kind: Event lastTimestamp: "2026-05-26T11:56:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:28Z" name: error-404-isvc-3c458-predictor.18b31b86bf311c00 namespace: kserve-ci-e2e-test resourceVersion: "23348" uid: adb3cef4-4e99-4e26-ab04-22f5b0ed2a13 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:55:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c458-predictor namespace: kserve-ci-e2e-test resourceVersion: "22708" uid: 808f594a-c5f1-45c4-8b08-0308a8e9aa00 kind: Event lastTimestamp: "2026-05-26T11:56:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:28Z" name: error-404-isvc-3c458-predictor.18b31b86bf962175 namespace: kserve-ci-e2e-test resourceVersion: "23349" uid: ec558baa-9ddc-4301-8178-24cf167f7e30 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c458-predictor namespace: kserve-ci-e2e-test resourceVersion: "22893" uid: 808f594a-c5f1-45c4-8b08-0308a8e9aa00 kind: Event lastTimestamp: "2026-05-26T11:56:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:56:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:43Z" name: error-404-isvc-3c458-predictor.18b31b94bac304a4 namespace: kserve-ci-e2e-test resourceVersion: "23481" uid: 58f013f3-9adb-4298-bf4f-5449909c6702 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c458-predictor namespace: kserve-ci-e2e-test resourceVersion: "22893" uid: 808f594a-c5f1-45c4-8b08-0308a8e9aa00 kind: Event lastTimestamp: "2026-05-26T11:56:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:56:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:43Z" name: error-404-isvc-3c458-predictor.18b31b94bb2aff1b namespace: kserve-ci-e2e-test resourceVersion: "23483" uid: 3d73b4aa-e7cb-44c9-832b-298b8e6896fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3c458 namespace: kserve-ci-e2e-test resourceVersion: "22675" uid: 50ac795e-ed71-40f2-9dc5-d0109284a606 kind: Event lastTimestamp: "2026-05-26T11:56:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:56:17Z" name: error-404-isvc-3c458.18b31b833f406a91 namespace: kserve-ci-e2e-test resourceVersion: "23236" uid: e163ce49-8069-47df-932b-c8c4e670bf8e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3c458 namespace: kserve-ci-e2e-test resourceVersion: "22675" uid: 50ac795e-ed71-40f2-9dc5-d0109284a606 kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: 'Failed to update status for InferenceService "error-404-isvc-3c458": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3c458": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:28Z" name: error-404-isvc-3c458.18b31b833fc7790d namespace: kserve-ci-e2e-test resourceVersion: "22714" uid: 000bcff0-f3cf-4a46-8b34-05f8ca585703 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3c458 namespace: kserve-ci-e2e-test resourceVersion: "22675" uid: 50ac795e-ed71-40f2-9dc5-d0109284a606 kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3c458": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:28Z" name: error-404-isvc-3c458.18b31b833fc7c5c1 namespace: kserve-ci-e2e-test resourceVersion: "22715" uid: cf776362-8b12-4c86-9950-f251f4e57d93 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3c458 namespace: kserve-ci-e2e-test resourceVersion: "23230" uid: 50ac795e-ed71-40f2-9dc5-d0109284a606 kind: Event lastTimestamp: "2026-05-26T11:56:16Z" message: InferenceService [error-404-isvc-3c458] is Ready metadata: creationTimestamp: "2026-05-26T11:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:56:16Z" name: error-404-isvc-3c458.18b31b8e93bbd8a8 namespace: kserve-ci-e2e-test resourceVersion: "23231" uid: d0a36f74-5e49-42bb-8542-6be83f339a47 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T12:05:23.815298Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5 namespace: kserve-ci-e2e-test resourceVersion: "27854" uid: 7c560caa-7292-4467-9656-8c68722a1166 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-52624-predictor-5c7c48695f-2g7f5 to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T12:05:23Z" name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5.18b31c0de945f8ea namespace: kserve-ci-e2e-test resourceVersion: "27858" uid: 49b166fd-aef3-4f39-91f0-f41db9a49847 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5 namespace: kserve-ci-e2e-test resourceVersion: "27856" uid: 7c560caa-7292-4467-9656-8c68722a1166 kind: Event lastTimestamp: "2026-05-26T12:05:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-52624-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:24Z" name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5.18b31c0df4887c13 namespace: kserve-ci-e2e-test resourceVersion: "27867" uid: 201d009f-9159-459c-8ed8-04c4fc875bb1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5 namespace: kserve-ci-e2e-test resourceVersion: "27864" uid: 7c560caa-7292-4467-9656-8c68722a1166 kind: Event lastTimestamp: "2026-05-26T12:05:24Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T12:05:24Z" name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5.18b31c0e26fe3a65 namespace: kserve-ci-e2e-test resourceVersion: "27888" uid: d1931abf-29a7-48cf-8dec-877634c0de20 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5 namespace: kserve-ci-e2e-test resourceVersion: "27856" uid: 7c560caa-7292-4467-9656-8c68722a1166 kind: Event lastTimestamp: "2026-05-26T12:05:24Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:24Z" name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5.18b31c0e287137d5 namespace: kserve-ci-e2e-test resourceVersion: "27893" uid: 3ea0a78d-daa0-4936-b65e-b22dca3a5922 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5 namespace: kserve-ci-e2e-test resourceVersion: "27856" uid: 7c560caa-7292-4467-9656-8c68722a1166 kind: Event lastTimestamp: "2026-05-26T12:05:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:24Z" name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5.18b31c0e2c04ddd2 namespace: kserve-ci-e2e-test resourceVersion: "27894" uid: f0ee3938-75e4-4aec-b161-af688a01aa13 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5 namespace: kserve-ci-e2e-test resourceVersion: "27856" uid: 7c560caa-7292-4467-9656-8c68722a1166 kind: Event lastTimestamp: "2026-05-26T12:05:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:24Z" name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5.18b31c0e2c7b9e47 namespace: kserve-ci-e2e-test resourceVersion: "27895" uid: 0fcf3bdd-38f9-4ca1-b05c-63e047c9aade reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5 namespace: kserve-ci-e2e-test resourceVersion: "27856" uid: 7c560caa-7292-4467-9656-8c68722a1166 kind: Event lastTimestamp: "2026-05-26T12:05:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:24Z" name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5.18b31c0e2c88ddd7 namespace: kserve-ci-e2e-test resourceVersion: "27896" uid: 6a0c022e-29c2-4527-a288-4a3e9905ea3a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5 namespace: kserve-ci-e2e-test resourceVersion: "27856" uid: 7c560caa-7292-4467-9656-8c68722a1166 kind: Event lastTimestamp: "2026-05-26T12:05:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T12:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:25Z" name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5.18b31c0e3042822e namespace: kserve-ci-e2e-test resourceVersion: "27901" uid: babf46b6-3ee6-4833-9ae5-f111e050efe6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5 namespace: kserve-ci-e2e-test resourceVersion: "27856" uid: 7c560caa-7292-4467-9656-8c68722a1166 kind: Event lastTimestamp: "2026-05-26T12:05:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:25Z" name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5.18b31c0e30bf7c66 namespace: kserve-ci-e2e-test resourceVersion: "27902" uid: 4ffc07dd-b533-4ce5-948e-1c33c59e25fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T12:05:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5 namespace: kserve-ci-e2e-test resourceVersion: "27856" uid: 7c560caa-7292-4467-9656-8c68722a1166 kind: Event lastTimestamp: "2026-05-26T12:14:41Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T12:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:14:41Z" name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5.18b31c0e51d51b6f namespace: kserve-ci-e2e-test resourceVersion: "31940" uid: bbe56785-a58d-4d84-9703-3e2c8b6112cf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5 namespace: kserve-ci-e2e-test resourceVersion: "27856" uid: 7c560caa-7292-4467-9656-8c68722a1166 kind: Event lastTimestamp: "2026-05-26T12:14:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T12:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:14:38Z" name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5.18b31c8f137638c4 namespace: kserve-ci-e2e-test resourceVersion: "31916" uid: 2e726e33-df39-4ac4-bbbb-76a75948fbbe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5 namespace: kserve-ci-e2e-test resourceVersion: "27856" uid: 7c560caa-7292-4467-9656-8c68722a1166 kind: Event lastTimestamp: "2026-05-26T12:14:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:14:38Z" name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5.18b31c8f1376a951 namespace: kserve-ci-e2e-test resourceVersion: "31917" uid: e1657507-7008-4058-b446-bd0efb7bb689 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:14:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5 namespace: kserve-ci-e2e-test resourceVersion: "27856" uid: 7c560caa-7292-4467-9656-8c68722a1166 kind: Event lastTimestamp: "2026-05-26T12:14:41Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T12:14:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:14:41Z" name: error-404-isvc-52624-predictor-5c7c48695f-2g7f5.18b31c8fc62a20ee namespace: kserve-ci-e2e-test resourceVersion: "31939" uid: fe8481c6-61b7-46c4-9b07-ebc9daabf922 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-52624-predictor-5c7c48695f namespace: kserve-ci-e2e-test resourceVersion: "27846" uid: be78d15d-0c79-4b87-8467-74890e8394f8 kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: 'Created pod: error-404-isvc-52624-predictor-5c7c48695f-2g7f5' metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:23Z" name: error-404-isvc-52624-predictor-5c7c48695f.18b31c0de7dd6d94 namespace: kserve-ci-e2e-test resourceVersion: "27857" uid: 2407683b-28b8-4bc7-8eb0-57b05e379440 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-52624-predictor namespace: kserve-ci-e2e-test resourceVersion: "27844" uid: b79b9d5e-96f6-4fa4-8804-27b83fab2785 kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: Scaled up replica set error-404-isvc-52624-predictor-5c7c48695f from 0 to 1 metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:23Z" name: error-404-isvc-52624-predictor.18b31c0de527b2a6 namespace: kserve-ci-e2e-test resourceVersion: "27850" uid: 8b82c8b8-c9fd-4ebf-b274-93fcd809bd1b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T12:05:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-52624-predictor namespace: kserve-ci-e2e-test resourceVersion: "27845" uid: 65cd5a9d-704e-4e5e-a69c-c2e02b9700e3 kind: Event lastTimestamp: "2026-05-26T12:06:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:06:08Z" name: error-404-isvc-52624-predictor.18b31c11639efc6b namespace: kserve-ci-e2e-test resourceVersion: "28328" uid: 9ace8324-cd25-4ad0-b8a5-85983110ffc5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T12:05:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-52624-predictor namespace: kserve-ci-e2e-test resourceVersion: "27845" uid: 65cd5a9d-704e-4e5e-a69c-c2e02b9700e3 kind: Event lastTimestamp: "2026-05-26T12:06:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:06:08Z" name: error-404-isvc-52624-predictor.18b31c116413fc14 namespace: kserve-ci-e2e-test resourceVersion: "28329" uid: e33f4d52-8951-4e28-aeaa-43844065794b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:06:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-52624-predictor namespace: kserve-ci-e2e-test resourceVersion: "28051" uid: 65cd5a9d-704e-4e5e-a69c-c2e02b9700e3 kind: Event lastTimestamp: "2026-05-26T12:06:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:06:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:06:38Z" name: error-404-isvc-52624-predictor.18b31c1be053b397 namespace: kserve-ci-e2e-test resourceVersion: "28604" uid: a1a8c5b8-9602-4f5e-b945-0007822814d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:06:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-52624-predictor namespace: kserve-ci-e2e-test resourceVersion: "28051" uid: 65cd5a9d-704e-4e5e-a69c-c2e02b9700e3 kind: Event lastTimestamp: "2026-05-26T12:06:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:06:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:06:38Z" name: error-404-isvc-52624-predictor.18b31c1be0c6c587 namespace: kserve-ci-e2e-test resourceVersion: "28605" uid: f40c88dd-fd64-481e-8c47-43e8c9ec9ad2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-52624 namespace: kserve-ci-e2e-test resourceVersion: "27814" uid: 5ab21e7a-2746-4717-8036-6fd151f69fbf kind: Event lastTimestamp: "2026-05-26T12:06:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:06:11Z" name: error-404-isvc-52624.18b31c0de4eee10b namespace: kserve-ci-e2e-test resourceVersion: "28364" uid: 0556a7a7-b7ab-4e3b-9370-50c8108b133c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-52624 namespace: kserve-ci-e2e-test resourceVersion: "27814" uid: 5ab21e7a-2746-4717-8036-6fd151f69fbf kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: 'Failed to update status for InferenceService "error-404-isvc-52624": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-52624": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:23Z" name: error-404-isvc-52624.18b31c0de55a1bcc namespace: kserve-ci-e2e-test resourceVersion: "27851" uid: 1184126d-563a-4195-8a62-8077327f9177 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-52624 namespace: kserve-ci-e2e-test resourceVersion: "27814" uid: 5ab21e7a-2746-4717-8036-6fd151f69fbf kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-52624": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:23Z" name: error-404-isvc-52624.18b31c0de55a6c69 namespace: kserve-ci-e2e-test resourceVersion: "27852" uid: 6f5c9520-2bae-4a9f-ae07-36aa37af6c21 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:06:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-52624 namespace: kserve-ci-e2e-test resourceVersion: "28362" uid: 5ab21e7a-2746-4717-8036-6fd151f69fbf kind: Event lastTimestamp: "2026-05-26T12:06:11Z" message: InferenceService [error-404-isvc-52624] is Ready metadata: creationTimestamp: "2026-05-26T12:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:06:11Z" name: error-404-isvc-52624.18b31c190d1d50b0 namespace: kserve-ci-e2e-test resourceVersion: "28363" uid: fc0497f4-36fe-4b2e-aac5-388954913e2d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:53:48.140247Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv namespace: kserve-ci-e2e-test resourceVersion: "21374" uid: c9325a47-b280-4387-89a6-5e0cba0b0aee kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5380a-predictor-78c9576cbc-hsjpv to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv.18b31b6befcf1974 namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 7695a824-421c-424b-9c47-6d5ca0d01fab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: c9325a47-b280-4387-89a6-5e0cba0b0aee kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv.18b31b6c09c674bb namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: d7bfdc4f-e3a2-42c6-8238-712aaad96594 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c9325a47-b280-4387-89a6-5e0cba0b0aee kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv.18b31b6c0b2d15e8 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: baaa6fe8-38f2-471c-be5b-729dacacf0ea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c9325a47-b280-4387-89a6-5e0cba0b0aee kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv.18b31b6c0e9007bb namespace: kserve-ci-e2e-test resourceVersion: "21404" uid: a9d96c9c-0434-43fc-9776-7b40abbba034 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c9325a47-b280-4387-89a6-5e0cba0b0aee kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv.18b31b6c0f065f79 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: cf9ff790-0f70-4193-b23d-bd374628c8eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c9325a47-b280-4387-89a6-5e0cba0b0aee kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv.18b31b6c0f147492 namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: ae56087f-4f75-493c-bc25-e26e2f3812c7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c9325a47-b280-4387-89a6-5e0cba0b0aee kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv.18b31b6c12c45b1c namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 57c76bc6-5308-4f21-9048-12bc849e4fcf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c9325a47-b280-4387-89a6-5e0cba0b0aee kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv.18b31b6c13426323 namespace: kserve-ci-e2e-test resourceVersion: "21408" uid: 2f30809b-b4c2-4023-9922-bd1f8ae1b473 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c9325a47-b280-4387-89a6-5e0cba0b0aee kind: Event lastTimestamp: "2026-05-26T11:54:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:26Z" name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv.18b31b6c7d183383 namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: 8ab0f957-dbe3-4bbc-8e7c-88362a92704d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c9325a47-b280-4387-89a6-5e0cba0b0aee kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:58Z" name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv.18b31b7c462a3ea4 namespace: kserve-ci-e2e-test resourceVersion: "22213" uid: 5db00348-f440-4f87-9e5c-122073354464 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c9325a47-b280-4387-89a6-5e0cba0b0aee kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:58Z" name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv.18b31b7c462b795c namespace: kserve-ci-e2e-test resourceVersion: "22216" uid: 59661f49-ed0e-4964-a50d-799647abe50f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c9325a47-b280-4387-89a6-5e0cba0b0aee kind: Event lastTimestamp: "2026-05-26T11:55:02Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-05-26T11:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:02Z" name: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv.18b31b7d410ec996 namespace: kserve-ci-e2e-test resourceVersion: "22355" uid: 6c49d6cd-9d74-43e8-b0a6-8ad8598f8347 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5380a-predictor-78c9576cbc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: f418647f-4b9c-4ee3-9f04-3679864d8117 kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: 'Created pod: error-404-isvc-5380a-predictor-78c9576cbc-hsjpv' metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-5380a-predictor-78c9576cbc.18b31b6bef517b4e namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: 9f176d73-c994-43ce-bfc2-712c531d7197 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5380a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: e6b37c16-cd36-4532-9bd8-9a58e8ad36cf kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: Scaled up replica set error-404-isvc-5380a-predictor-78c9576cbc from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-5380a-predictor.18b31b6bed105245 namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: 6d9ceeed-55dc-4e2a-8128-ff0b108421aa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5380a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: 7d8ac543-2799-4b6e-bee5-6e0f28cd2343 kind: Event lastTimestamp: "2026-05-26T11:54:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:03Z" name: error-404-isvc-5380a-predictor.18b31b6f6bcf2265 namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: 0c1cce83-b705-4f08-9942-70795938ba95 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5380a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: 7d8ac543-2799-4b6e-bee5-6e0f28cd2343 kind: Event lastTimestamp: "2026-05-26T11:54:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:03Z" name: error-404-isvc-5380a-predictor.18b31b6f6c39be3a namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: f61ef645-2667-4e2d-9e12-e6dce7372d43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:54:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5380a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: 7d8ac543-2799-4b6e-bee5-6e0f28cd2343 kind: Event lastTimestamp: "2026-05-26T11:54:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:48Z" name: error-404-isvc-5380a-predictor.18b31b72eada697d namespace: kserve-ci-e2e-test resourceVersion: "22084" uid: 35ab1576-a074-4e6b-be01-d1dbb4101dda reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:54:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5380a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: 7d8ac543-2799-4b6e-bee5-6e0f28cd2343 kind: Event lastTimestamp: "2026-05-26T11:54:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:48Z" name: error-404-isvc-5380a-predictor.18b31b72eb4001bd namespace: kserve-ci-e2e-test resourceVersion: "22085" uid: 5dd06b4d-2405-493d-9601-a42bd8a015f9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5380a namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: f36f098f-6c04-409b-9ca9-8cfb770baedb kind: Event lastTimestamp: "2026-05-26T11:54:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:36Z" name: error-404-isvc-5380a.18b31b6beceace92 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: 4bea5618-f3d4-4f2e-856b-eee354830d96 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5380a namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: f36f098f-6c04-409b-9ca9-8cfb770baedb kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: 'Failed to update status for InferenceService "error-404-isvc-5380a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5380a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-5380a.18b31b6bed7ffddc namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: 298f53b3-9382-4c49-8ac7-c62dab974889 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5380a namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: f36f098f-6c04-409b-9ca9-8cfb770baedb kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5380a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-5380a.18b31b6bed806be7 namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: afac8883-0979-42bc-855b-c6f2fd6480ec reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5380a namespace: kserve-ci-e2e-test resourceVersion: "21966" uid: f36f098f-6c04-409b-9ca9-8cfb770baedb kind: Event lastTimestamp: "2026-05-26T11:54:36Z" message: InferenceService [error-404-isvc-5380a] is Ready metadata: creationTimestamp: "2026-05-26T11:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:36Z" name: error-404-isvc-5380a.18b31b773a26ea21 namespace: kserve-ci-e2e-test resourceVersion: "21967" uid: d2167255-302e-4db4-8e14-89e525df299c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:45:03.096738Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g namespace: kserve-ci-e2e-test resourceVersion: "17469" uid: d85374dd-6464-461d-9e24-50bd9911b878 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5b0de-predictor-6766f587d9-qlg2g to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g.18b31af1b0c6f9d4 namespace: kserve-ci-e2e-test resourceVersion: "17473" uid: 477d954f-c462-4443-ba3f-1ea9fcb4698c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g namespace: kserve-ci-e2e-test resourceVersion: "17479" uid: d85374dd-6464-461d-9e24-50bd9911b878 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g.18b31af1caeabe8a namespace: kserve-ci-e2e-test resourceVersion: "17491" uid: 9dc6eeaa-d598-46ad-b168-351744616de4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: d85374dd-6464-461d-9e24-50bd9911b878 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g.18b31af1cc4025cd namespace: kserve-ci-e2e-test resourceVersion: "17494" uid: 87f1df04-8d89-46c9-af93-8dc9356b4d64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: d85374dd-6464-461d-9e24-50bd9911b878 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g.18b31af1cfd214fe namespace: kserve-ci-e2e-test resourceVersion: "17496" uid: 64c4d07c-79fd-4d84-a0ab-3ceab430ab36 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: d85374dd-6464-461d-9e24-50bd9911b878 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g.18b31af1d0585588 namespace: kserve-ci-e2e-test resourceVersion: "17497" uid: 2bca7ba8-4103-47f5-8664-6825fef7e6a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: d85374dd-6464-461d-9e24-50bd9911b878 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g.18b31af1d065866e namespace: kserve-ci-e2e-test resourceVersion: "17498" uid: 8e2dd699-15d8-4744-97d1-6f93da354727 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: d85374dd-6464-461d-9e24-50bd9911b878 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g.18b31af1d4f6afb8 namespace: kserve-ci-e2e-test resourceVersion: "17501" uid: be7a9c21-10a1-4694-bd95-5aa6eb9a0d2d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: d85374dd-6464-461d-9e24-50bd9911b878 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g.18b31af1d57cae7f namespace: kserve-ci-e2e-test resourceVersion: "17503" uid: 08bcb9b9-3bbf-4e88-b63e-e4f668437353 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: d85374dd-6464-461d-9e24-50bd9911b878 kind: Event lastTimestamp: "2026-05-26T11:54:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:20Z" name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g.18b31af1e62a799e namespace: kserve-ci-e2e-test resourceVersion: "21808" uid: b481771f-2a82-4fae-8ab1-9890d56cf5ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: d85374dd-6464-461d-9e24-50bd9911b878 kind: Event lastTimestamp: "2026-05-26T11:54:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:17Z" name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g.18b31b72e1e91777 namespace: kserve-ci-e2e-test resourceVersion: "21728" uid: 09adee97-e1ba-4765-9de0-9150ef1b2a66 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: d85374dd-6464-461d-9e24-50bd9911b878 kind: Event lastTimestamp: "2026-05-26T11:54:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:17Z" name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g.18b31b72e1e98c96 namespace: kserve-ci-e2e-test resourceVersion: "21730" uid: 1b718514-6649-4b5f-a6bd-275b5f3d8d78 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g namespace: kserve-ci-e2e-test resourceVersion: "17470" uid: d85374dd-6464-461d-9e24-50bd9911b878 kind: Event lastTimestamp: "2026-05-26T11:54:19Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:54:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:19Z" name: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g.18b31b735aaf827e namespace: kserve-ci-e2e-test resourceVersion: "21806" uid: 9ec658a6-b66c-49be-a389-774cf1c3c6e4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5b0de-predictor-6766f587d9 namespace: kserve-ci-e2e-test resourceVersion: "17460" uid: 4e4b8223-cb8c-4b9a-bb83-af1bd4f51620 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: 'Created pod: error-404-isvc-5b0de-predictor-6766f587d9-qlg2g' metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-5b0de-predictor-6766f587d9.18b31af1b0643a71 namespace: kserve-ci-e2e-test resourceVersion: "17471" uid: 696200c3-360a-42e9-83e2-75d3aed62ddb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5b0de-predictor namespace: kserve-ci-e2e-test resourceVersion: "17458" uid: ce098ad7-b057-4a30-b6c1-404dbf7d75e9 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: Scaled up replica set error-404-isvc-5b0de-predictor-6766f587d9 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-5b0de-predictor.18b31af1ae8501a2 namespace: kserve-ci-e2e-test resourceVersion: "17462" uid: d0890817-5f78-4c6e-b4c8-08c7d3157dd8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:45:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5b0de-predictor namespace: kserve-ci-e2e-test resourceVersion: "17459" uid: acc3cebf-5f06-4ef3-80af-89385c3322b9 kind: Event lastTimestamp: "2026-05-26T11:45:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:33Z" name: error-404-isvc-5b0de-predictor.18b31af52d2d025d namespace: kserve-ci-e2e-test resourceVersion: "17789" uid: fed00abc-bd8b-4eb8-8e0e-1b276005b4e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:45:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5b0de-predictor namespace: kserve-ci-e2e-test resourceVersion: "17459" uid: acc3cebf-5f06-4ef3-80af-89385c3322b9 kind: Event lastTimestamp: "2026-05-26T11:45:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:33Z" name: error-404-isvc-5b0de-predictor.18b31af52d95aba2 namespace: kserve-ci-e2e-test resourceVersion: "17791" uid: b3ba237d-bd43-44a8-99bb-c1cb12effeb0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:45:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5b0de-predictor namespace: kserve-ci-e2e-test resourceVersion: "17648" uid: acc3cebf-5f06-4ef3-80af-89385c3322b9 kind: Event lastTimestamp: "2026-05-26T11:46:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:18Z" name: error-404-isvc-5b0de-predictor.18b31afc2b3ed1f6 namespace: kserve-ci-e2e-test resourceVersion: "18228" uid: 4b5c9c1b-34d4-4bb0-adc4-a54733ffea92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:45:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5b0de-predictor namespace: kserve-ci-e2e-test resourceVersion: "17648" uid: acc3cebf-5f06-4ef3-80af-89385c3322b9 kind: Event lastTimestamp: "2026-05-26T11:46:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:18Z" name: error-404-isvc-5b0de-predictor.18b31afc2b88e615 namespace: kserve-ci-e2e-test resourceVersion: "18229" uid: 94f5a545-d5ec-4ced-8a02-187624dd98d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5b0de namespace: kserve-ci-e2e-test resourceVersion: "17427" uid: 0e029fc2-d9bb-4f5e-a9f8-1e40c7f4f2fd kind: Event lastTimestamp: "2026-05-26T11:45:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:50Z" name: error-404-isvc-5b0de.18b31af1ae6067d0 namespace: kserve-ci-e2e-test resourceVersion: "17973" uid: 5986476b-f65a-4153-9c72-0e803de5e236 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5b0de namespace: kserve-ci-e2e-test resourceVersion: "17427" uid: 0e029fc2-d9bb-4f5e-a9f8-1e40c7f4f2fd kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: 'Failed to update status for InferenceService "error-404-isvc-5b0de": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5b0de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-5b0de.18b31af1aeb4aa1b namespace: kserve-ci-e2e-test resourceVersion: "17464" uid: d04f8d5b-28eb-4b46-9024-94ec9a5a2d7c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5b0de namespace: kserve-ci-e2e-test resourceVersion: "17427" uid: 0e029fc2-d9bb-4f5e-a9f8-1e40c7f4f2fd kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5b0de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-5b0de.18b31af1aeb516bf namespace: kserve-ci-e2e-test resourceVersion: "17466" uid: b54a6d14-006d-4f1f-8c34-99a7e03e32d4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5b0de namespace: kserve-ci-e2e-test resourceVersion: "17970" uid: 0e029fc2-d9bb-4f5e-a9f8-1e40c7f4f2fd kind: Event lastTimestamp: "2026-05-26T11:45:50Z" message: InferenceService [error-404-isvc-5b0de] is Ready metadata: creationTimestamp: "2026-05-26T11:45:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:50Z" name: error-404-isvc-5b0de.18b31afca17161d4 namespace: kserve-ci-e2e-test resourceVersion: "17971" uid: 841983f5-93ac-4dc4-a3ac-c2ebb866c12e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T12:04:13.489004Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m namespace: kserve-ci-e2e-test resourceVersion: "26945" uid: 0462e858-a74c-468c-a60d-71276b56c110 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-81833-predictor-6db55d4ff5-xfl9m to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T12:04:13Z" name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m.18b31bfd898008bb namespace: kserve-ci-e2e-test resourceVersion: "26949" uid: 47019c7a-f532-4d21-992d-56902272c424 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 0462e858-a74c-468c-a60d-71276b56c110 kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-81833-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:13Z" name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m.18b31bfd90d76a76 namespace: kserve-ci-e2e-test resourceVersion: "26960" uid: af66ef13-2209-4575-bdf5-8bb6c780adef reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m namespace: kserve-ci-e2e-test resourceVersion: "26956" uid: 0462e858-a74c-468c-a60d-71276b56c110 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T12:04:14Z" name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m.18b31bfdc71fc57e namespace: kserve-ci-e2e-test resourceVersion: "26981" uid: e92430ea-9702-4f0a-8341-83086e510c1b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 0462e858-a74c-468c-a60d-71276b56c110 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:14Z" name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m.18b31bfdc8b2e0c8 namespace: kserve-ci-e2e-test resourceVersion: "26986" uid: e0a987ef-5806-46f3-be32-c4e5e26e807a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 0462e858-a74c-468c-a60d-71276b56c110 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:14Z" name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m.18b31bfdcba2de17 namespace: kserve-ci-e2e-test resourceVersion: "26987" uid: 22cc1821-7e58-42a3-8e05-13fef8cf3a2e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 0462e858-a74c-468c-a60d-71276b56c110 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:14Z" name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m.18b31bfdcc247af5 namespace: kserve-ci-e2e-test resourceVersion: "26988" uid: d06bd478-6785-4cc4-b1a0-14db0ccc277b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 0462e858-a74c-468c-a60d-71276b56c110 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:14Z" name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m.18b31bfdcc320cec namespace: kserve-ci-e2e-test resourceVersion: "26989" uid: b7699dc3-ca48-4cbd-8a29-a18c390bd837 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 0462e858-a74c-468c-a60d-71276b56c110 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:14Z" name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m.18b31bfdcf734dd9 namespace: kserve-ci-e2e-test resourceVersion: "26992" uid: f0c118d1-47cb-4235-83ea-d1f28312da3f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 0462e858-a74c-468c-a60d-71276b56c110 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:14Z" name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m.18b31bfdcff94b98 namespace: kserve-ci-e2e-test resourceVersion: "26993" uid: 172b6580-647c-457b-b22c-7bc22a863a96 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T12:04:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 0462e858-a74c-468c-a60d-71276b56c110 kind: Event lastTimestamp: "2026-05-26T12:04:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T12:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:51Z" name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m.18b31bfdf7c6b06a namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: aac5b0ed-acc9-4c41-9f39-18a63f83d25e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 0462e858-a74c-468c-a60d-71276b56c110 kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:23Z" name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m.18b31c0de33ee5d2 namespace: kserve-ci-e2e-test resourceVersion: "27839" uid: 694ed4d0-407b-427a-a9ad-f210fc407672 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 0462e858-a74c-468c-a60d-71276b56c110 kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:23Z" name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m.18b31c0de33f7852 namespace: kserve-ci-e2e-test resourceVersion: "27841" uid: 821bff18-3d79-4000-aad0-d7d40043922c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 0462e858-a74c-468c-a60d-71276b56c110 kind: Event lastTimestamp: "2026-05-26T12:05:26Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T12:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:26Z" name: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m.18b31c0e7fe3a18d namespace: kserve-ci-e2e-test resourceVersion: "27917" uid: 20b290ad-dabd-4311-906d-14eb1ee827f9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-81833-predictor-6db55d4ff5 namespace: kserve-ci-e2e-test resourceVersion: "26937" uid: 188f7057-23c9-4fed-8c6d-cc16e60e4bf0 kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: 'Created pod: error-404-isvc-81833-predictor-6db55d4ff5-xfl9m' metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:04:13Z" name: error-404-isvc-81833-predictor-6db55d4ff5.18b31bfd890b3c49 namespace: kserve-ci-e2e-test resourceVersion: "26946" uid: 3f3e9016-d266-492a-89b2-297d12cbf867 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-81833-predictor namespace: kserve-ci-e2e-test resourceVersion: "26933" uid: c5a896c7-f349-4fa2-ac27-80dd5b776755 kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: Scaled up replica set error-404-isvc-81833-predictor-6db55d4ff5 from 0 to 1 metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:04:13Z" name: error-404-isvc-81833-predictor.18b31bfd86e49dab namespace: kserve-ci-e2e-test resourceVersion: "26939" uid: 45607c4e-371f-44b4-9988-dac3799100e3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:04:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-81833-predictor namespace: kserve-ci-e2e-test resourceVersion: "26936" uid: c460c8fa-8839-4fd8-902c-79404681952e kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:04:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-81833-predictor.18b31c01057f55c6 namespace: kserve-ci-e2e-test resourceVersion: "27347" uid: 68af0cee-c61c-4e2b-9381-5c6bbbf157d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:04:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-81833-predictor namespace: kserve-ci-e2e-test resourceVersion: "26936" uid: c460c8fa-8839-4fd8-902c-79404681952e kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:04:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:04:43Z" name: error-404-isvc-81833-predictor.18b31c0105efb094 namespace: kserve-ci-e2e-test resourceVersion: "27349" uid: 1dc19469-1635-40b6-b4da-8218a49506ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:04:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-81833-predictor namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: c460c8fa-8839-4fd8-902c-79404681952e kind: Event lastTimestamp: "2026-05-26T12:05:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:13Z" name: error-404-isvc-81833-predictor.18b31c0803bb6752 namespace: kserve-ci-e2e-test resourceVersion: "27663" uid: f0f81f7d-1925-4292-b610-bd9e0d4f371e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:04:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-81833-predictor namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: c460c8fa-8839-4fd8-902c-79404681952e kind: Event lastTimestamp: "2026-05-26T12:05:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:13Z" name: error-404-isvc-81833-predictor.18b31c08044a5dc4 namespace: kserve-ci-e2e-test resourceVersion: "27665" uid: 916fc6f4-420b-49ae-b7a1-72c13e7d2fe1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-81833 namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: ac221546-a7f1-4a7d-b3fa-63545fed1e85 kind: Event lastTimestamp: "2026-05-26T12:05:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:01Z" name: error-404-isvc-81833.18b31bfd86c9ca68 namespace: kserve-ci-e2e-test resourceVersion: "27539" uid: 3deba913-d86a-43d3-ba12-dc4007470956 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-81833 namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: ac221546-a7f1-4a7d-b3fa-63545fed1e85 kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: 'Failed to update status for InferenceService "error-404-isvc-81833": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-81833": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:04:13Z" name: error-404-isvc-81833.18b31bfd873fb1c3 namespace: kserve-ci-e2e-test resourceVersion: "26941" uid: cab0d631-f7ff-416b-8a64-cadd1412db31 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-81833 namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: ac221546-a7f1-4a7d-b3fa-63545fed1e85 kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-81833": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:04:13Z" name: error-404-isvc-81833.18b31bfd873ffe89 namespace: kserve-ci-e2e-test resourceVersion: "26943" uid: 217dc4f6-63fe-4e2c-a85c-08f1a34b18b5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-81833 namespace: kserve-ci-e2e-test resourceVersion: "27536" uid: ac221546-a7f1-4a7d-b3fa-63545fed1e85 kind: Event lastTimestamp: "2026-05-26T12:05:01Z" message: InferenceService [error-404-isvc-81833] is Ready metadata: creationTimestamp: "2026-05-26T12:05:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:01Z" name: error-404-isvc-81833.18b31c08b271eded namespace: kserve-ci-e2e-test resourceVersion: "27537" uid: 67ebdcf7-1172-4149-85a2-a48517a11d05 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:44:33.176416Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-90450-predictor-f47cbcc5c-5s579 namespace: kserve-ci-e2e-test resourceVersion: "17013" uid: bb0ee137-b57f-4795-b42f-c3f26a00639e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-90450-predictor-f47cbcc5c-5s579 to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-90450-predictor-f47cbcc5c-5s579.18b31aeab9631728 namespace: kserve-ci-e2e-test resourceVersion: "17017" uid: 931f80b0-3a43-499f-9f51-e39c34d43d33 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-90450-predictor-f47cbcc5c-5s579 namespace: kserve-ci-e2e-test resourceVersion: "17024" uid: bb0ee137-b57f-4795-b42f-c3f26a00639e kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-90450-predictor-f47cbcc5c-5s579.18b31aead2d3b9fb namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 4b6edc1e-b1f8-4fc2-8a75-11976e12aec6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-90450-predictor-f47cbcc5c-5s579 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: bb0ee137-b57f-4795-b42f-c3f26a00639e kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-90450-predictor-f47cbcc5c-5s579.18b31aead42a647b namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: 7fdddabd-f008-48bf-a8e8-2ae3c9b6d94e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-90450-predictor-f47cbcc5c-5s579 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: bb0ee137-b57f-4795-b42f-c3f26a00639e kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-90450-predictor-f47cbcc5c-5s579.18b31aead6dd6287 namespace: kserve-ci-e2e-test resourceVersion: "17044" uid: d87b0dac-baaa-404c-a8a9-8df72ab78e7b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-90450-predictor-f47cbcc5c-5s579 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: bb0ee137-b57f-4795-b42f-c3f26a00639e kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-90450-predictor-f47cbcc5c-5s579.18b31aead757d281 namespace: kserve-ci-e2e-test resourceVersion: "17045" uid: d98bb011-ccc7-4334-856c-e25ea119caba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-90450-predictor-f47cbcc5c-5s579 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: bb0ee137-b57f-4795-b42f-c3f26a00639e kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-90450-predictor-f47cbcc5c-5s579.18b31aead765661e namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: caff4b82-b362-42d1-9753-3671e24a93ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-90450-predictor-f47cbcc5c-5s579 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: bb0ee137-b57f-4795-b42f-c3f26a00639e kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-90450-predictor-f47cbcc5c-5s579.18b31aeada5691b5 namespace: kserve-ci-e2e-test resourceVersion: "17047" uid: b5c4979a-96a7-4ae5-b4e7-cda202fef1d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-90450-predictor-f47cbcc5c-5s579 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: bb0ee137-b57f-4795-b42f-c3f26a00639e kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-90450-predictor-f47cbcc5c-5s579.18b31aeadad4522c namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 8d769ca5-1fc3-4702-8c64-0d53630527e2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-90450-predictor-f47cbcc5c-5s579 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: bb0ee137-b57f-4795-b42f-c3f26a00639e kind: Event lastTimestamp: "2026-05-26T11:53:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:49Z" name: error-404-isvc-90450-predictor-f47cbcc5c-5s579.18b31aeae447e1cf namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: a9bab226-b4de-4687-9938-dac65e6a00fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-90450-predictor-f47cbcc5c-5s579 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: bb0ee137-b57f-4795-b42f-c3f26a00639e kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-90450-predictor-f47cbcc5c-5s579.18b31b6beb2d5824 namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: 82d9bf87-7955-4420-af8c-6ade9ad47454 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-90450-predictor-f47cbcc5c-5s579 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: bb0ee137-b57f-4795-b42f-c3f26a00639e kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: error-404-isvc-90450-predictor-f47cbcc5c-5s579.18b31b6beb2dbf22 namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: 83d4d651-ba7d-4769-8df5-8091c777feb9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-90450-predictor-f47cbcc5c-5s579 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: bb0ee137-b57f-4795-b42f-c3f26a00639e kind: Event lastTimestamp: "2026-05-26T11:53:49Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:53:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:49Z" name: error-404-isvc-90450-predictor-f47cbcc5c-5s579.18b31b6c58b65739 namespace: kserve-ci-e2e-test resourceVersion: "21425" uid: fbf64581-2a62-4e8b-9604-fd89a026ccf0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-90450-predictor-f47cbcc5c namespace: kserve-ci-e2e-test resourceVersion: "17006" uid: 1aa19a17-6df5-4a65-9de4-a9aac286359e kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: 'Created pod: error-404-isvc-90450-predictor-f47cbcc5c-5s579' metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-90450-predictor-f47cbcc5c.18b31aeab8f2304a namespace: kserve-ci-e2e-test resourceVersion: "17014" uid: dab13172-692d-4bb6-8e57-b4f5d5066c06 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-90450-predictor namespace: kserve-ci-e2e-test resourceVersion: "17004" uid: 1fcc0e80-f1fd-494b-9bed-33dce5ab616e kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: Scaled up replica set error-404-isvc-90450-predictor-f47cbcc5c from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-90450-predictor.18b31aeab772c8b2 namespace: kserve-ci-e2e-test resourceVersion: "17008" uid: 8f97239a-088f-4960-9d22-d115effee80c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:44:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-90450-predictor namespace: kserve-ci-e2e-test resourceVersion: "17005" uid: 71cb7791-344c-407f-93b4-f0961a216f23 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-90450-predictor.18b31aee360f792b namespace: kserve-ci-e2e-test resourceVersion: "17481" uid: 4bc20991-1242-4a90-8f85-d80c47e12e65 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:44:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-90450-predictor namespace: kserve-ci-e2e-test resourceVersion: "17005" uid: 71cb7791-344c-407f-93b4-f0961a216f23 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:03Z" name: error-404-isvc-90450-predictor.18b31aee3676107c namespace: kserve-ci-e2e-test resourceVersion: "17483" uid: 6f22d4be-cfee-485e-8d82-c5d375e2413a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:45:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-90450-predictor namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: 71cb7791-344c-407f-93b4-f0961a216f23 kind: Event lastTimestamp: "2026-05-26T11:45:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:48Z" name: error-404-isvc-90450-predictor.18b31af5340503fc namespace: kserve-ci-e2e-test resourceVersion: "17950" uid: 59e797ee-3bd1-4c81-837f-99174cb14c69 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:45:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-90450-predictor namespace: kserve-ci-e2e-test resourceVersion: "17194" uid: 71cb7791-344c-407f-93b4-f0961a216f23 kind: Event lastTimestamp: "2026-05-26T11:45:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:48Z" name: error-404-isvc-90450-predictor.18b31af5346cf7bb namespace: kserve-ci-e2e-test resourceVersion: "17951" uid: 699db4d1-1292-4961-a289-d5a38eed66c2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-90450 namespace: kserve-ci-e2e-test resourceVersion: "16984" uid: 6ddde9e7-0b80-48b9-8db9-6d531f439500 kind: Event lastTimestamp: "2026-05-26T11:45:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:20Z" name: error-404-isvc-90450.18b31aeab747bbae namespace: kserve-ci-e2e-test resourceVersion: "17675" uid: 99609c6b-7f65-4d92-be62-682529a03884 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-90450 namespace: kserve-ci-e2e-test resourceVersion: "16984" uid: 6ddde9e7-0b80-48b9-8db9-6d531f439500 kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: 'Failed to update status for InferenceService "error-404-isvc-90450": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-90450": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-90450.18b31aeab7eb02c6 namespace: kserve-ci-e2e-test resourceVersion: "17010" uid: 86069ffd-63c2-4167-85ce-914bd99fff8f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-90450 namespace: kserve-ci-e2e-test resourceVersion: "16984" uid: 6ddde9e7-0b80-48b9-8db9-6d531f439500 kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-90450": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:33Z" name: error-404-isvc-90450.18b31aeab7eb58d1 namespace: kserve-ci-e2e-test resourceVersion: "17011" uid: 0d2a4181-37ff-495d-b298-9972fa4f4f4a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-90450 namespace: kserve-ci-e2e-test resourceVersion: "17672" uid: 6ddde9e7-0b80-48b9-8db9-6d531f439500 kind: Event lastTimestamp: "2026-05-26T11:45:19Z" message: InferenceService [error-404-isvc-90450] is Ready metadata: creationTimestamp: "2026-05-26T11:45:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:19Z" name: error-404-isvc-90450.18b31af59f4ab52f namespace: kserve-ci-e2e-test resourceVersion: "17673" uid: d640ef49-cd9c-4691-a04a-f89200ddfa1f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:54:18.034349Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb namespace: kserve-ci-e2e-test resourceVersion: "21743" uid: b280eea3-83f6-44fe-9ec8-3566464fb8ee kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:54:18Z" name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb.18b31b72e5a2e8c0 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: 26fd5e09-0b44-4a05-952f-739db18bce3c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:18Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: b280eea3-83f6-44fe-9ec8-3566464fb8ee kind: Event lastTimestamp: "2026-05-26T11:54:18Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:54:18Z" name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb.18b31b72ff67a3a1 namespace: kserve-ci-e2e-test resourceVersion: "21772" uid: 19454197-0185-42a3-a21e-861ff7a4bd74 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: b280eea3-83f6-44fe-9ec8-3566464fb8ee kind: Event lastTimestamp: "2026-05-26T11:54:18Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:18Z" name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb.18b31b7300e243ab namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: 7a3a9f94-48a1-4bc9-8703-df51414d520a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: b280eea3-83f6-44fe-9ec8-3566464fb8ee kind: Event lastTimestamp: "2026-05-26T11:54:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:18Z" name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb.18b31b7303f75592 namespace: kserve-ci-e2e-test resourceVersion: "21775" uid: 8d0a27f6-33f7-4681-96df-5e473653873a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: b280eea3-83f6-44fe-9ec8-3566464fb8ee kind: Event lastTimestamp: "2026-05-26T11:54:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:18Z" name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb.18b31b73046f5ec7 namespace: kserve-ci-e2e-test resourceVersion: "21777" uid: fa5eb6f3-44a6-4656-8302-e8d6d4bbef47 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: b280eea3-83f6-44fe-9ec8-3566464fb8ee kind: Event lastTimestamp: "2026-05-26T11:54:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:18Z" name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb.18b31b73047b171e namespace: kserve-ci-e2e-test resourceVersion: "21778" uid: 85b8a469-e7bd-42e1-b8f3-79251ef333ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: b280eea3-83f6-44fe-9ec8-3566464fb8ee kind: Event lastTimestamp: "2026-05-26T11:54:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:18Z" name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb.18b31b73079e13fe namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: de2909ce-e5e4-46f3-a6a2-62746fbd7f14 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: b280eea3-83f6-44fe-9ec8-3566464fb8ee kind: Event lastTimestamp: "2026-05-26T11:54:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:18Z" name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb.18b31b730858da21 namespace: kserve-ci-e2e-test resourceVersion: "21783" uid: 6e71357b-47d7-41f0-a346-ba8f6ca2fdc3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: b280eea3-83f6-44fe-9ec8-3566464fb8ee kind: Event lastTimestamp: "2026-05-26T11:54:55Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:55Z" name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb.18b31b734379f271 namespace: kserve-ci-e2e-test resourceVersion: "22151" uid: 3d848e15-2f71-4948-827a-6cbedb2c569a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: b280eea3-83f6-44fe-9ec8-3566464fb8ee kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:28Z" name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb.18b31b833eadc4b2 namespace: kserve-ci-e2e-test resourceVersion: "22707" uid: c652e472-9c14-4536-89ef-056dc61b1f53 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: b280eea3-83f6-44fe-9ec8-3566464fb8ee kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:28Z" name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb.18b31b833eae4b8b namespace: kserve-ci-e2e-test resourceVersion: "22710" uid: 49c1a0ee-815e-461e-9b51-720b8ca52259 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb namespace: kserve-ci-e2e-test resourceVersion: "21745" uid: b280eea3-83f6-44fe-9ec8-3566464fb8ee kind: Event lastTimestamp: "2026-05-26T11:55:30Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:30Z" name: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb.18b31b83cb898054 namespace: kserve-ci-e2e-test resourceVersion: "22783" uid: 977ab02a-4f22-4e63-a052-99553ba2afba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fbdd2-predictor-5d98fbb9bf namespace: kserve-ci-e2e-test resourceVersion: "21735" uid: 301328ab-b1bd-40e4-99b7-f83a6bbdf981 kind: Event lastTimestamp: "2026-05-26T11:54:18Z" message: 'Created pod: error-404-isvc-fbdd2-predictor-5d98fbb9bf-p4zsb' metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:18Z" name: error-404-isvc-fbdd2-predictor-5d98fbb9bf.18b31b72e53637a1 namespace: kserve-ci-e2e-test resourceVersion: "21744" uid: 5901e239-7014-4c8c-91cc-9889947f2b0c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fbdd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21733" uid: 6fe29137-8e30-4867-aa09-c1a9a8d24b44 kind: Event lastTimestamp: "2026-05-26T11:54:17Z" message: Scaled up replica set error-404-isvc-fbdd2-predictor-5d98fbb9bf from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:17Z" name: error-404-isvc-fbdd2-predictor.18b31b72e359d3ae namespace: kserve-ci-e2e-test resourceVersion: "21737" uid: 4e1229b6-3200-4a0e-b50e-37a219b10366 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:54:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbdd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21734" uid: bf0f981a-6515-42a0-84bf-f760cd450581 kind: Event lastTimestamp: "2026-05-26T11:55:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:03Z" name: error-404-isvc-fbdd2-predictor.18b31b7661f5ce26 namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 59315f9b-2637-4ab3-8ed8-41a55c24ae82 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:54:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbdd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21734" uid: bf0f981a-6515-42a0-84bf-f760cd450581 kind: Event lastTimestamp: "2026-05-26T11:55:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:54:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:03Z" name: error-404-isvc-fbdd2-predictor.18b31b766241a2c1 namespace: kserve-ci-e2e-test resourceVersion: "22362" uid: b9948d12-e051-46bf-ada4-269405c6f8db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbdd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21913" uid: bf0f981a-6515-42a0-84bf-f760cd450581 kind: Event lastTimestamp: "2026-05-26T11:55:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:18Z" name: error-404-isvc-fbdd2-predictor.18b31b80deaa396a namespace: kserve-ci-e2e-test resourceVersion: "22528" uid: 64bd3416-b809-4e0d-9522-a8cebb413bc6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fbdd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21913" uid: bf0f981a-6515-42a0-84bf-f760cd450581 kind: Event lastTimestamp: "2026-05-26T11:55:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:18Z" name: error-404-isvc-fbdd2-predictor.18b31b80df109b01 namespace: kserve-ci-e2e-test resourceVersion: "22531" uid: b5dd7670-7641-42ff-a123-cc578cd5dc75 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbdd2 namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: 68483133-a737-44b2-adf3-36e52ffbd573 kind: Event lastTimestamp: "2026-05-26T11:55:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:05Z" name: error-404-isvc-fbdd2.18b31b72e33b279b namespace: kserve-ci-e2e-test resourceVersion: "22389" uid: 3dc0108a-0d1f-4c02-bf88-34173194edc1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbdd2 namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: 68483133-a737-44b2-adf3-36e52ffbd573 kind: Event lastTimestamp: "2026-05-26T11:54:17Z" message: 'Failed to update status for InferenceService "error-404-isvc-fbdd2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fbdd2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:18Z" name: error-404-isvc-fbdd2.18b31b72e3882c64 namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: b583f1d7-d84e-4f94-8ceb-bb4525484e67 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbdd2 namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: 68483133-a737-44b2-adf3-36e52ffbd573 kind: Event lastTimestamp: "2026-05-26T11:54:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fbdd2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:18Z" name: error-404-isvc-fbdd2.18b31b72e3888a95 namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: 899e1fa7-1c49-4466-81f1-259de10c2664 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fbdd2 namespace: kserve-ci-e2e-test resourceVersion: "22386" uid: 68483133-a737-44b2-adf3-36e52ffbd573 kind: Event lastTimestamp: "2026-05-26T11:55:05Z" message: InferenceService [error-404-isvc-fbdd2] is Ready metadata: creationTimestamp: "2026-05-26T11:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:05Z" name: error-404-isvc-fbdd2.18b31b7dfed38ac8 namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 3e18116d-7b19-4898-b72b-8729a8ac9787 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:43:02.967933Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-678784f46b-8m468 to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:43:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:43:02Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ad5b88aeb14 namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 4a68db8b-a289-4ddc-a9b8-46b00b2a5eb0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:43:03Z" message: Add eth0 [10.134.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:43:03Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ad5d1fc5d8d namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 420c900d-d63d-4452-a3fb-b7488545d77b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:43:03Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:03Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ad5d363ffde namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: 0e464fca-dd0e-4a09-9de4-19fdccf9377e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:43:08Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" in 4.893s (4.893s including waiting). Image size: 303109559 bytes.' metadata: creationTimestamp: "2026-05-26T11:43:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:08Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ad6f70b069b namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: 9a584439-e0e2-4a04-b143-311cbe37051c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:43:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:43:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:08Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ad6faac31f1 namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: c8051402-215e-4ba6-bd32-57e4615a850d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:43:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:43:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:08Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ad6fb35afd3 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 7bacbf37-9d7d-4b11-bdb1-9a7d9c8d91ee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:43:12Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" metadata: creationTimestamp: "2026-05-26T11:43:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:12Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ad7f967254c namespace: kserve-ci-e2e-test resourceVersion: "16114" uid: b8702534-ac6f-4c0a-b536-7cba2c77c589 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:43:22Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" in 9.418s (9.418s including waiting). Image size: 1560928695 bytes.' metadata: creationTimestamp: "2026-05-26T11:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:22Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ada2ac8b05a namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: 02245300-a532-4227-b9e1-14e46a5d123f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:43:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:22Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ada2e7c7276 namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: bf59fdbf-a3ee-4e44-aa36-76791f79bb8a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:43:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:22Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ada2ef6167c namespace: kserve-ci-e2e-test resourceVersion: "16243" uid: 661db679-58e8-479d-9ed8-9639898924b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:43:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:22Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ada2f0c6056 namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: 9a2d2221-a6d3-4ca9-9208-13a0c44bac19 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:43:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:22Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ada31f8ae87 namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: 13248bb6-d214-4872-8967-f0dc09c3fd9f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:43:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:22Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ada3273307b namespace: kserve-ci-e2e-test resourceVersion: "16246" uid: 247f85d8-c2ba-4645-954d-6986f033baba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-26T11:43:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:44:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:43:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:18Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31ada4fad9458 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: fcaca3db-8c5e-472f-a1fc-09e4237734b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:03Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31af1ac08c822 namespace: kserve-ci-e2e-test resourceVersion: "17449" uid: c94e0a29-0829-4aed-89c6-d031890469c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:03Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31af1ac0960b7 namespace: kserve-ci-e2e-test resourceVersion: "17452" uid: f4439c7b-8721-4252-abc7-f9a4cac46873 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-678784f46b-8m468 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 0805d6c1-804e-4bac-93cc-ea3e36a86829 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: 'Readiness probe failed: Get "https://10.134.0.18:8643/healthz": dial tcp 10.134.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:03Z" name: isvc-sklearn-graph-1-predictor-678784f46b-8m468.18b31af1d3dda025 namespace: kserve-ci-e2e-test resourceVersion: "17500" uid: bdf0aeff-e202-42ff-8474-04f47d64a701 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-678784f46b namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 68a560fa-a093-4430-bab7-714c335f14b4 kind: Event lastTimestamp: "2026-05-26T11:43:02Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-678784f46b-8m468' metadata: creationTimestamp: "2026-05-26T11:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:02Z" name: isvc-sklearn-graph-1-predictor-678784f46b.18b31ad5b7f7b3c0 namespace: kserve-ci-e2e-test resourceVersion: "15920" uid: 77e66a18-183b-4025-9333-76e7906dc994 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: 394f0ae4-d6f3-4ce2-a698-72057dc96d98 kind: Event lastTimestamp: "2026-05-26T11:43:02Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-678784f46b from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:02Z" name: isvc-sklearn-graph-1-predictor.18b31ad5b1d9fafb namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: 773264be-defb-4d3b-a1ba-6187cacbcfe9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:43:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: f4956546-ceb6-4be8-afb3-fdeb72559c82 kind: Event lastTimestamp: "2026-05-26T11:44:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:02Z" name: isvc-sklearn-graph-1-predictor.18b31ad930262c8f namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: 94e7511f-7116-41cc-955a-2cb77ce1d717 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:43:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: f4956546-ceb6-4be8-afb3-fdeb72559c82 kind: Event lastTimestamp: "2026-05-26T11:44:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:02Z" name: isvc-sklearn-graph-1-predictor.18b31ad930a709dc namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: 78350109-f8bf-4e8b-9096-2b63a67083c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:43:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: f4956546-ceb6-4be8-afb3-fdeb72559c82 kind: Event lastTimestamp: "2026-05-26T11:44:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:43:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:47Z" name: isvc-sklearn-graph-1-predictor.18b31ae02e2a70f0 namespace: kserve-ci-e2e-test resourceVersion: "17187" uid: f86fdbfa-4818-47a3-9d53-f90bf6237d8b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:43:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: f4956546-ceb6-4be8-afb3-fdeb72559c82 kind: Event lastTimestamp: "2026-05-26T11:44:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:43:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:47Z" name: isvc-sklearn-graph-1-predictor.18b31ae02e6d309d namespace: kserve-ci-e2e-test resourceVersion: "17188" uid: 60bcccd3-48f1-44b0-bf8c-ec96f1ded83d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: f4956546-ceb6-4be8-afb3-fdeb72559c82 kind: Event lastTimestamp: "2026-05-26T11:45:02Z" message: 'Operation cannot be fulfilled on horizontalpodautoscalers.autoscaling "isvc-sklearn-graph-1-predictor": StorageError: invalid object, Code: 4, Key: /kubernetes.io/horizontalpodautoscalers/kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: f4956546-ceb6-4be8-afb3-fdeb72559c82, UID in object meta: ' metadata: creationTimestamp: "2026-05-26T11:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:02Z" name: isvc-sklearn-graph-1-predictor.18b31af1aa008d72 namespace: kserve-ci-e2e-test resourceVersion: "17432" uid: f7bac1ef-7f7a-478a-8777-9e26d49f0e29 reason: FailedUpdateStatus reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-26T11:43:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: a407ae86-6321-4f3e-bb47-6aab4127db0c kind: Event lastTimestamp: "2026-05-26T11:44:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:28Z" name: isvc-sklearn-graph-1.18b31ad5b15e59a8 namespace: kserve-ci-e2e-test resourceVersion: "16871" uid: 1bba11fe-8621-41a7-b42e-17ff08109be4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16869" uid: a407ae86-6321-4f3e-bb47-6aab4127db0c kind: Event lastTimestamp: "2026-05-26T11:44:28Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-05-26T11:44:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:28Z" name: isvc-sklearn-graph-1.18b31ae9b4139893 namespace: kserve-ci-e2e-test resourceVersion: "16870" uid: b2d67c43-d566-4c58-8c5f-43227ee97b4d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:43:03.822276Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p to ip-10-0-135-71.ec2.internal metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:43:03Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31ad5eb7717e0 namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 90501731-037c-4474-9a3d-bd8d9d9fcb92 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:43:04Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:43:04Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:43:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:04Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31ad5f902f41f namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: c87db2a2-d06e-4531-9313-7680d2dc856b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:43:05Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:43:05Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31ad66355b536 namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: 468a6dd9-432d-445a-8655-7f73c0766cf8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:43:05Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" metadata: creationTimestamp: "2026-05-26T11:43:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:05Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31ad6652112b1 namespace: kserve-ci-e2e-test resourceVersion: "16041" uid: d5cd7020-8d4b-4a47-b1b6-e16f94198c4a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:43:15Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" in 9.644s (9.644s including waiting). Image size: 303109559 bytes.' metadata: creationTimestamp: "2026-05-26T11:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:15Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31ad8a3f6de73 namespace: kserve-ci-e2e-test resourceVersion: "16146" uid: 03cadba1-9449-4be6-a290-aabb76b03895 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:43:15Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:15Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31ad8a7f9d639 namespace: kserve-ci-e2e-test resourceVersion: "16147" uid: 7ad20d69-b4f0-4fc8-a407-4eb23405af5e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:43:15Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:15Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31ad8a86a182c namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: f4575a53-db55-4425-b0ae-42e250b26865 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:43:19Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" metadata: creationTimestamp: "2026-05-26T11:43:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:19Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31ad98573c0e2 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: a8cf5dbb-9459-484f-8729-17e9b35db6d4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:43:24Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1532" in 5.626s (5.626s including waiting). Image size: 1560928695 bytes.' metadata: creationTimestamp: "2026-05-26T11:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:24Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31adad4d390c9 namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: d6b2258f-5036-41ea-b7b4-7106212dbfc3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:43:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:24Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31adad83e8f52 namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 7a77661d-80be-4500-9f6e-1291edf3341a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:43:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:24Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31adad8adac0c namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: cd484e2c-3eda-4c4b-a70d-ace755c3e39a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:43:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:43:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:24Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31adad8c4bd03 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: c36670da-665f-4467-8ce7-8124adfcbf5d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:43:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:25Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31adadc7fb42d namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: d46e9023-7654-48ff-b335-7c444a00e966 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:43:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:43:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:25Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31adadcf2b8e4 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: dff12614-be41-47da-ace7-792211671352 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-05-26T11:43:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:44:22Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:43:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:22Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31adb287f278b namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: 0c4ff607-292f-4813-97ab-9c3523ed86f5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:45:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:02Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31af1a9758e81 namespace: kserve-ci-e2e-test resourceVersion: "17421" uid: 60e52dda-501c-4a9f-ab3a-b7d8b91e5bd5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:45:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:02Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31af1a97873b5 namespace: kserve-ci-e2e-test resourceVersion: "17429" uid: 29bbbdaf-5c90-4b64-a8bd-530b7f29074a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: e9dff39d-f2cb-48e9-a3c1-3753107c084c kind: Event lastTimestamp: "2026-05-26T11:45:07Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:45:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:07Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p.18b31af2acbd8372 namespace: kserve-ci-e2e-test resourceVersion: "17554" uid: 5353ec4d-1406-404c-928a-81689c3f82f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-54c99bfc7d namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: b987c770-dab3-482d-ba59-533b22b83283 kind: Event lastTimestamp: "2026-05-26T11:43:03Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-54c99bfc7d-fjv9p' metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:03Z" name: isvc-sklearn-graph-2-predictor-54c99bfc7d.18b31ad5eb0ec644 namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 92304dca-0d4f-471b-a203-9c4f29df13e9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: e8585cdd-98ce-4277-a5f4-11232cb8b28b kind: Event lastTimestamp: "2026-05-26T11:43:03Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-54c99bfc7d from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:03Z" name: isvc-sklearn-graph-2-predictor.18b31ad5d45aa206 namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: b3067870-dcf5-4cf2-b844-e04a06effd53 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:43:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: ef1ea548-3c33-498b-a640-bc70c32831ae kind: Event lastTimestamp: "2026-05-26T11:44:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:03Z" name: isvc-sklearn-graph-2-predictor.18b31ad953198d91 namespace: kserve-ci-e2e-test resourceVersion: "16604" uid: d109f728-8e36-4bfa-94a4-3b90bd9c40f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:43:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: ef1ea548-3c33-498b-a640-bc70c32831ae kind: Event lastTimestamp: "2026-05-26T11:44:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:03Z" name: isvc-sklearn-graph-2-predictor.18b31ad95387a9f2 namespace: kserve-ci-e2e-test resourceVersion: "16605" uid: 74a548f5-4731-41ea-896c-1d3ea28f19db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:43:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: ef1ea548-3c33-498b-a640-bc70c32831ae kind: Event lastTimestamp: "2026-05-26T11:44:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:43:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:48Z" name: isvc-sklearn-graph-2-predictor.18b31ae05130e259 namespace: kserve-ci-e2e-test resourceVersion: "17201" uid: f80d9daf-aa53-4981-bd8d-55ec41e17cc9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:43:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: ef1ea548-3c33-498b-a640-bc70c32831ae kind: Event lastTimestamp: "2026-05-26T11:44:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:43:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:48Z" name: isvc-sklearn-graph-2-predictor.18b31ae051ada798 namespace: kserve-ci-e2e-test resourceVersion: "17203" uid: 03abbf72-8f03-4513-8c12-e265fac9c291 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 92c23c86-0791-4732-9d12-5b2dc3dc4449 kind: Event lastTimestamp: "2026-05-26T11:44:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:32Z" name: isvc-sklearn-graph-2.18b31ad5d43c846f namespace: kserve-ci-e2e-test resourceVersion: "16910" uid: e36bafe7-e01d-4ca1-b573-368d494e67ae reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: 92c23c86-0791-4732-9d12-5b2dc3dc4449 kind: Event lastTimestamp: "2026-05-26T11:44:32Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-05-26T11:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:32Z" name: isvc-sklearn-graph-2.18b31aea8ba6db70 namespace: kserve-ci-e2e-test resourceVersion: "16909" uid: d3b749ce-0207-47c2-9d4d-35432a65da2b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:43:03.217730Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-ptkzq to ip-10-0-133-249.ec2.internal metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:43:03Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31ad5c76ead62 namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: b4817cce-7303-4e19-aa8b-ede95909f1b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:43:03Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:43:03Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31ad5e0ba8d2a namespace: kserve-ci-e2e-test resourceVersion: "15994" uid: 47fad533-3011-4017-a4d3-25fbe745cd0f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:43:03Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:03Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31ad5e23309d7 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 2329a5fc-39fa-4e42-b558-9722387479ab reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:43:06Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:30bffa55f5e3e0e2f6d023c0f84846c206091122d724f5a0e810b4fb957dd389" in 3.167s (3.167s including waiting). Image size: 303109559 bytes.' metadata: creationTimestamp: "2026-05-26T11:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31ad69f070e43 namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: cdc2312b-4f86-4899-9be2-cb64aecb9d2d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:43:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-05-26T11:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31ad6a20f0b1e namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: a4f4114e-4610-432a-8452-078a75920c89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:43:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-05-26T11:43:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:06Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31ad6a27e58d2 namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: 4b3b4064-27ca-4693-b65a-8d2074b74976 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:43:10Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-05-26T11:43:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:10Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31ad78a9c7336 namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 691056c6-de21-4b04-9867-2061fdd7cd5e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:43:28Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 17.812s (17.812s including waiting). Image size: 1306238203 bytes.' metadata: creationTimestamp: "2026-05-26T11:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:28Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31adbb0568bfd namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: b857f479-9725-46af-b2bb-f89d506843a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:43:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:28Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31adbb37c4ee7 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: 76de6963-1007-46fa-8789-c2a44dbc232f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:43:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:28Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31adbb3ed6dd9 namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: 67f5f63e-138d-4d97-b6bd-774969a0a6c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:43:28Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-26T11:43:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:28Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31adbb3f94da8 namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: cee459e1-fb06-48f9-bc25-db69c3737507 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:43:30Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.109s (2.109s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-26T11:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:30Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31adc31aeb7d6 namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: 85774541-9e47-407c-8637-363ea91925ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:43:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:30Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31adc34ed8358 namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 3de5199d-74ca-40a4-9764-efbe67d1292a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:43:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:43:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:30Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31adc355f738c namespace: kserve-ci-e2e-test resourceVersion: "16331" uid: 5d2c6210-b66e-4553-b71d-c2e1e0b12df1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T11:43:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:44:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:43:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:17Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31adc72a4e28b namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 593f75b8-a9c1-4c76-a896-1a6ca4f09b9b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:45:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:02Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31af1a4fbd285 namespace: kserve-ci-e2e-test resourceVersion: "17390" uid: 12d0d0fb-58fa-485d-bdc0-53a529cfc828 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 97a68551-9ff9-4f31-9b1e-2ae813f1156b kind: Event lastTimestamp: "2026-05-26T11:45:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:02Z" name: isvc-xgboost-graph-predictor-669d8d6456-ptkzq.18b31af1a4fc5e31 namespace: kserve-ci-e2e-test resourceVersion: "17393" uid: 4cf77015-1911-49d3-9ddb-c138020e6ff5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-249.ec2.internal source: component: kubelet host: ip-10-0-133-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: b1698730-c8f3-44fe-afe6-8aee328f39f5 kind: Event lastTimestamp: "2026-05-26T11:43:03Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-ptkzq' metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:03Z" name: isvc-xgboost-graph-predictor-669d8d6456.18b31ad5c6fc1ff3 namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: 85fe7db3-af6e-4c1e-be6e-b2c85f23c5ac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: a254f454-25dd-4332-8226-58656179e13e kind: Event lastTimestamp: "2026-05-26T11:43:03Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:03Z" name: isvc-xgboost-graph-predictor.18b31ad5bfda1265 namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: 76c94119-7114-4046-8ef8-40f5e44a166d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:43:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15950" uid: 324da758-d2ec-4546-a481-1cd0e1d40c25 kind: Event lastTimestamp: "2026-05-26T11:44:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:03Z" name: isvc-xgboost-graph-predictor.18b31ad93e612558 namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: 3c340630-b591-42d0-8262-2e6fc04b70ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:43:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15950" uid: 324da758-d2ec-4546-a481-1cd0e1d40c25 kind: Event lastTimestamp: "2026-05-26T11:44:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:03Z" name: isvc-xgboost-graph-predictor.18b31ad93ece526b namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: 11066f76-f80b-45c7-a846-ca82becfc741 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 324da758-d2ec-4546-a481-1cd0e1d40c25 kind: Event lastTimestamp: "2026-05-26T11:44:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:48Z" name: isvc-xgboost-graph-predictor.18b31ae73a0e6983 namespace: kserve-ci-e2e-test resourceVersion: "17197" uid: de11ba79-964c-429e-84e8-f3b49ebc3f8e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:44:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16194" uid: 324da758-d2ec-4546-a481-1cd0e1d40c25 kind: Event lastTimestamp: "2026-05-26T11:44:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:44:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:48Z" name: isvc-xgboost-graph-predictor.18b31ae73a8e7368 namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 999a172f-af94-4b4a-8e31-178cc5a8abb2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: 5280f34f-ae6e-44d1-a7bf-6638a53f6702 kind: Event lastTimestamp: "2026-05-26T11:44:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:27Z" name: isvc-xgboost-graph.18b31ad5bf984b7e namespace: kserve-ci-e2e-test resourceVersion: "16857" uid: bc52116c-9a2c-4dfa-9cce-655597acec55 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16855" uid: 5280f34f-ae6e-44d1-a7bf-6638a53f6702 kind: Event lastTimestamp: "2026-05-26T11:44:27Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-05-26T11:44:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:27Z" name: isvc-xgboost-graph.18b31ae981b2153d namespace: kserve-ci-e2e-test resourceVersion: "16856" uid: f8093207-1954-43b8-bac5-a4f0a2886408 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:44:52.824373Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-866798f95b-zp2x7 namespace: kserve-ci-e2e-test resourceVersion: "17245" uid: 27d83a5b-252b-4c74-9487-6656ec81ec5a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-866798f95b-zp2x7 to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:44:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:44:52Z" name: model-chainer-866798f95b-zp2x7.18b31aef4c7f21f7 namespace: kserve-ci-e2e-test resourceVersion: "17250" uid: a62ac901-05cb-4372-8483-c75dfe3d64a3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:53Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-866798f95b-zp2x7 namespace: kserve-ci-e2e-test resourceVersion: "17253" uid: 27d83a5b-252b-4c74-9487-6656ec81ec5a kind: Event lastTimestamp: "2026-05-26T11:44:53Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:44:53Z" name: model-chainer-866798f95b-zp2x7.18b31aef66280656 namespace: kserve-ci-e2e-test resourceVersion: "17268" uid: d29f5e88-d228-4284-9024-534c868221fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-866798f95b-zp2x7 namespace: kserve-ci-e2e-test resourceVersion: "17246" uid: 27d83a5b-252b-4c74-9487-6656ec81ec5a kind: Event lastTimestamp: "2026-05-26T11:44:53Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-26T11:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:53Z" name: model-chainer-866798f95b-zp2x7.18b31aef6774b6b4 namespace: kserve-ci-e2e-test resourceVersion: "17270" uid: f1c78011-7956-4cd9-b1b2-f52ec8bea736 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-866798f95b-zp2x7 namespace: kserve-ci-e2e-test resourceVersion: "17246" uid: 27d83a5b-252b-4c74-9487-6656ec81ec5a kind: Event lastTimestamp: "2026-05-26T11:44:53Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-05-26T11:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:53Z" name: model-chainer-866798f95b-zp2x7.18b31aef6ae74ff5 namespace: kserve-ci-e2e-test resourceVersion: "17271" uid: a2f9ba5b-a649-41d5-9fb0-f5e692f6f83c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-866798f95b-zp2x7 namespace: kserve-ci-e2e-test resourceVersion: "17246" uid: 27d83a5b-252b-4c74-9487-6656ec81ec5a kind: Event lastTimestamp: "2026-05-26T11:44:53Z" message: Started container model-chainer metadata: creationTimestamp: "2026-05-26T11:44:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:53Z" name: model-chainer-866798f95b-zp2x7.18b31aef6b6589ed namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: ef4cb6f4-1603-4072-81e6-d0f7b85e82ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-866798f95b-zp2x7 namespace: kserve-ci-e2e-test resourceVersion: "17246" uid: 27d83a5b-252b-4c74-9487-6656ec81ec5a kind: Event lastTimestamp: "2026-05-26T11:45:02Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-05-26T11:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:02Z" name: model-chainer-866798f95b-zp2x7.18b31af1a015b8fb namespace: kserve-ci-e2e-test resourceVersion: "17369" uid: f99d670d-ab4f-4694-b93f-581753bab4e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-866798f95b-zp2x7 namespace: kserve-ci-e2e-test resourceVersion: "17246" uid: 27d83a5b-252b-4c74-9487-6656ec81ec5a kind: Event lastTimestamp: "2026-05-26T11:45:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T11:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:29Z" name: model-chainer-866798f95b-zp2x7.18b31af2202ac156 namespace: kserve-ci-e2e-test resourceVersion: "17756" uid: 771a0390-c5d3-4e7e-ac4f-6318c890bf6b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-866798f95b namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: a3e8a122-bfe3-46d9-81d2-eaf5c06e7f56 kind: Event lastTimestamp: "2026-05-26T11:44:52Z" message: 'Created pod: model-chainer-866798f95b-zp2x7' metadata: creationTimestamp: "2026-05-26T11:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:52Z" name: model-chainer-866798f95b.18b31aef4c07af29 namespace: kserve-ci-e2e-test resourceVersion: "17248" uid: 157d08ee-f09a-4ee0-9ef0-18eef36d3411 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17240" uid: 84892a25-d423-46d8-8336-44ed1f330dd5 kind: Event lastTimestamp: "2026-05-26T11:44:52Z" message: Scaled up replica set model-chainer-866798f95b from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:52Z" name: model-chainer.18b31aef4b265b9e namespace: kserve-ci-e2e-test resourceVersion: "17243" uid: 6d0c0ff9-d9a6-4d07-bc2e-0c754b7259fb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17258" uid: ad4f8e02-fdb1-4eeb-b8f7-d4c130cc0a88 kind: Event lastTimestamp: "2026-05-26T11:44:52Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-05-26T11:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:52Z" name: model-chainer.18b31aef512041fc namespace: kserve-ci-e2e-test resourceVersion: "17260" uid: b1dd37ff-0ce0-4a61-8c43-8ff8a5d718b7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17233" uid: ad4f8e02-fdb1-4eeb-b8f7-d4c130cc0a88 kind: Event lastTimestamp: "2026-05-26T11:44:52Z" message: 'Failed to update status for InferenceGraph "model-chainer": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:52Z" name: model-chainer.18b31aef53cd8fc1 namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: 2627ec82-3de1-4220-9895-df576c933f98 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17233" uid: ad4f8e02-fdb1-4eeb-b8f7-d4c130cc0a88 kind: Event lastTimestamp: "2026-05-26T11:44:52Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "model-chainer": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:44:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:52Z" name: model-chainer.18b31aef53cde7a1 namespace: kserve-ci-e2e-test resourceVersion: "17264" uid: 5c7c5b21-0ef6-4c7d-a07d-c0a35709d7e9 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:56:28.266537Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3c458-6bcbbc655c-q2t5j namespace: kserve-ci-e2e-test resourceVersion: "23333" uid: 3bab8610-ad1e-4408-8df0-13d0cfbd7440 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-3c458-6bcbbc655c-q2t5j to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:56:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:56:28Z" name: sequence-graph-3c458-6bcbbc655c-q2t5j.18b31b9138145b59 namespace: kserve-ci-e2e-test resourceVersion: "23338" uid: f55bed0e-cdd1-4039-9b00-831927f33f34 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:28Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3c458-6bcbbc655c-q2t5j namespace: kserve-ci-e2e-test resourceVersion: "23335" uid: 3bab8610-ad1e-4408-8df0-13d0cfbd7440 kind: Event lastTimestamp: "2026-05-26T11:56:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-3c458-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:56:28Z" name: sequence-graph-3c458-6bcbbc655c-q2t5j.18b31b9143ef566b namespace: kserve-ci-e2e-test resourceVersion: "23358" uid: 71a67c7e-f2fa-4fac-9eda-01f965c82cdf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:29Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3c458-6bcbbc655c-q2t5j namespace: kserve-ci-e2e-test resourceVersion: "23342" uid: 3bab8610-ad1e-4408-8df0-13d0cfbd7440 kind: Event lastTimestamp: "2026-05-26T11:56:29Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:56:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:56:29Z" name: sequence-graph-3c458-6bcbbc655c-q2t5j.18b31b91754a9bce namespace: kserve-ci-e2e-test resourceVersion: "23366" uid: 7c0a0298-4575-4e23-ba13-a089337b3ebd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3c458} kind: Pod name: sequence-graph-3c458-6bcbbc655c-q2t5j namespace: kserve-ci-e2e-test resourceVersion: "23335" uid: 3bab8610-ad1e-4408-8df0-13d0cfbd7440 kind: Event lastTimestamp: "2026-05-26T11:56:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-26T11:56:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:56:29Z" name: sequence-graph-3c458-6bcbbc655c-q2t5j.18b31b9176ab9fd2 namespace: kserve-ci-e2e-test resourceVersion: "23368" uid: 1058a406-ab70-45c8-a159-6832594a4c35 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3c458} kind: Pod name: sequence-graph-3c458-6bcbbc655c-q2t5j namespace: kserve-ci-e2e-test resourceVersion: "23335" uid: 3bab8610-ad1e-4408-8df0-13d0cfbd7440 kind: Event lastTimestamp: "2026-05-26T11:56:29Z" message: 'Created container: sequence-graph-3c458' metadata: creationTimestamp: "2026-05-26T11:56:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:56:29Z" name: sequence-graph-3c458-6bcbbc655c-q2t5j.18b31b9179cdae16 namespace: kserve-ci-e2e-test resourceVersion: "23369" uid: de917366-bb45-4a88-b6ae-d516112589c8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3c458} kind: Pod name: sequence-graph-3c458-6bcbbc655c-q2t5j namespace: kserve-ci-e2e-test resourceVersion: "23335" uid: 3bab8610-ad1e-4408-8df0-13d0cfbd7440 kind: Event lastTimestamp: "2026-05-26T11:56:29Z" message: Started container sequence-graph-3c458 metadata: creationTimestamp: "2026-05-26T11:56:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:56:29Z" name: sequence-graph-3c458-6bcbbc655c-q2t5j.18b31b917a54bf35 namespace: kserve-ci-e2e-test resourceVersion: "23370" uid: fa5d9dd3-e52b-4bd1-8632-a2076afc9bb3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3c458} kind: Pod name: sequence-graph-3c458-6bcbbc655c-q2t5j namespace: kserve-ci-e2e-test resourceVersion: "23335" uid: 3bab8610-ad1e-4408-8df0-13d0cfbd7440 kind: Event lastTimestamp: "2026-05-26T12:04:42Z" message: Stopping container sequence-graph-3c458 metadata: creationTimestamp: "2026-05-26T12:04:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:42Z" name: sequence-graph-3c458-6bcbbc655c-q2t5j.18b31c045bd2832a namespace: kserve-ci-e2e-test resourceVersion: "27247" uid: 1a8d4f2b-a41b-483b-acfc-2bb52d4248e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T12:04:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3c458} kind: Pod name: sequence-graph-3c458-6bcbbc655c-q2t5j namespace: kserve-ci-e2e-test resourceVersion: "23335" uid: 3bab8610-ad1e-4408-8df0-13d0cfbd7440 kind: Event lastTimestamp: "2026-05-26T12:05:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T12:04:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:11Z" name: sequence-graph-3c458-6bcbbc655c-q2t5j.18b31c051c368b7a namespace: kserve-ci-e2e-test resourceVersion: "27618" uid: 15807448-2793-4c01-af5e-395b616a8350 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-3c458-6bcbbc655c namespace: kserve-ci-e2e-test resourceVersion: "23329" uid: 3c97d824-185b-4bf2-bf9e-e1327c8d3db7 kind: Event lastTimestamp: "2026-05-26T11:56:28Z" message: 'Created pod: sequence-graph-3c458-6bcbbc655c-q2t5j' metadata: creationTimestamp: "2026-05-26T11:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:28Z" name: sequence-graph-3c458-6bcbbc655c.18b31b9137b49001 namespace: kserve-ci-e2e-test resourceVersion: "23334" uid: cb985990-8f9b-42a7-837a-fabb33668b5b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-3c458 namespace: kserve-ci-e2e-test resourceVersion: "23326" uid: 0150b4b0-f4e5-446e-94fc-4a079c1302b9 kind: Event lastTimestamp: "2026-05-26T11:56:28Z" message: Scaled up replica set sequence-graph-3c458-6bcbbc655c from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:28Z" name: sequence-graph-3c458.18b31b9136f0a428 namespace: kserve-ci-e2e-test resourceVersion: "23331" uid: af7115be-ec66-4cb3-a4be-0da3578fbbb8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-3c458 namespace: kserve-ci-e2e-test resourceVersion: "23345" uid: 9bac9016-4cbb-4ef1-85cf-d8893a33fe45 kind: Event lastTimestamp: "2026-05-26T11:56:28Z" message: InferenceGraph [sequence-graph-3c458] is Ready metadata: creationTimestamp: "2026-05-26T11:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:56:28Z" name: sequence-graph-3c458.18b31b913c01e32f namespace: kserve-ci-e2e-test resourceVersion: "23347" uid: 1ba49df5-3aeb-4344-a29d-1b75e6a80047 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:56:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-3c458 namespace: kserve-ci-e2e-test resourceVersion: "23328" uid: e8763454-88f4-43c6-84b0-144e615923e9 kind: Event lastTimestamp: "2026-05-26T11:56:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:56:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:58Z" name: sequence-graph-3c458.18b31b94b55dc938 namespace: kserve-ci-e2e-test resourceVersion: "23584" uid: 0a5759f1-33c9-4dbf-9ddf-243d88b014a9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:56:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-3c458 namespace: kserve-ci-e2e-test resourceVersion: "23328" uid: e8763454-88f4-43c6-84b0-144e615923e9 kind: Event lastTimestamp: "2026-05-26T11:56:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:56:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:58Z" name: sequence-graph-3c458.18b31b94b5bd3900 namespace: kserve-ci-e2e-test resourceVersion: "23585" uid: d90fb607-562c-4157-9f5e-147d7e242889 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:46:02.994441Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-5b0de-94f97df94-8xmqh namespace: kserve-ci-e2e-test resourceVersion: "18087" uid: 1f724af6-d324-41d0-9ccb-12cdda5bc341 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-5b0de-94f97df94-8xmqh to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:46:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:46:02Z" name: sequence-graph-5b0de-94f97df94-8xmqh.18b31affa2f560da namespace: kserve-ci-e2e-test resourceVersion: "18092" uid: a16008f4-2de5-4abd-955e-427bfa8e87ff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:03Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-5b0de-94f97df94-8xmqh namespace: kserve-ci-e2e-test resourceVersion: "18095" uid: 1f724af6-d324-41d0-9ccb-12cdda5bc341 kind: Event lastTimestamp: "2026-05-26T11:46:03Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:46:03Z" name: sequence-graph-5b0de-94f97df94-8xmqh.18b31affbc5df63f namespace: kserve-ci-e2e-test resourceVersion: "18114" uid: 7e6c8b45-1e94-4162-89a2-85361f75ea60 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5b0de} kind: Pod name: sequence-graph-5b0de-94f97df94-8xmqh namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: 1f724af6-d324-41d0-9ccb-12cdda5bc341 kind: Event lastTimestamp: "2026-05-26T11:46:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-26T11:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:03Z" name: sequence-graph-5b0de-94f97df94-8xmqh.18b31affbd993159 namespace: kserve-ci-e2e-test resourceVersion: "18116" uid: c53ca153-96f3-4efb-a4b9-641084e8edc3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5b0de} kind: Pod name: sequence-graph-5b0de-94f97df94-8xmqh namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: 1f724af6-d324-41d0-9ccb-12cdda5bc341 kind: Event lastTimestamp: "2026-05-26T11:46:03Z" message: 'Created container: sequence-graph-5b0de' metadata: creationTimestamp: "2026-05-26T11:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:03Z" name: sequence-graph-5b0de-94f97df94-8xmqh.18b31affc13a979f namespace: kserve-ci-e2e-test resourceVersion: "18117" uid: 0958a804-fe7a-4148-963c-53e15c7a3b2b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5b0de} kind: Pod name: sequence-graph-5b0de-94f97df94-8xmqh namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: 1f724af6-d324-41d0-9ccb-12cdda5bc341 kind: Event lastTimestamp: "2026-05-26T11:46:03Z" message: Started container sequence-graph-5b0de metadata: creationTimestamp: "2026-05-26T11:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:46:03Z" name: sequence-graph-5b0de-94f97df94-8xmqh.18b31affc1b4a4bc namespace: kserve-ci-e2e-test resourceVersion: "18118" uid: 2e9d906f-a8c2-4a06-aad4-4426d4c8118d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5b0de} kind: Pod name: sequence-graph-5b0de-94f97df94-8xmqh namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: 1f724af6-d324-41d0-9ccb-12cdda5bc341 kind: Event lastTimestamp: "2026-05-26T11:54:17Z" message: Stopping container sequence-graph-5b0de metadata: creationTimestamp: "2026-05-26T11:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:17Z" name: sequence-graph-5b0de-94f97df94-8xmqh.18b31b72d6f26f0b namespace: kserve-ci-e2e-test resourceVersion: "21667" uid: 1fe29c64-7bb2-41c3-8f1a-4e6b67ed8d3d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:54:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5b0de} kind: Pod name: sequence-graph-5b0de-94f97df94-8xmqh namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: 1f724af6-d324-41d0-9ccb-12cdda5bc341 kind: Event lastTimestamp: "2026-05-26T11:54:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T11:54:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:45Z" name: sequence-graph-5b0de-94f97df94-8xmqh.18b31b7365d7b11a namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: 0336759f-15dd-43aa-bc9d-3222f2b5669d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-5b0de-94f97df94 namespace: kserve-ci-e2e-test resourceVersion: "18084" uid: bdf20681-92a0-4f94-9671-a28a0dc23767 kind: Event lastTimestamp: "2026-05-26T11:46:02Z" message: 'Created pod: sequence-graph-5b0de-94f97df94-8xmqh' metadata: creationTimestamp: "2026-05-26T11:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:02Z" name: sequence-graph-5b0de-94f97df94.18b31affa27f6b72 namespace: kserve-ci-e2e-test resourceVersion: "18088" uid: 85093a26-0fe1-4073-bbb4-06e96be53856 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-5b0de namespace: kserve-ci-e2e-test resourceVersion: "18082" uid: 35943f90-4ff7-40d4-8b66-5c9943a35ca3 kind: Event lastTimestamp: "2026-05-26T11:46:02Z" message: Scaled up replica set sequence-graph-5b0de-94f97df94 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:02Z" name: sequence-graph-5b0de.18b31affa1c7bdbb namespace: kserve-ci-e2e-test resourceVersion: "18085" uid: 1344a5e3-88c8-4fb9-a4c0-32571f8e4cc2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-5b0de namespace: kserve-ci-e2e-test resourceVersion: "18100" uid: 2b758c57-8257-4f0f-b08a-7279cb0bc43d kind: Event lastTimestamp: "2026-05-26T11:46:03Z" message: InferenceGraph [sequence-graph-5b0de] is Ready metadata: creationTimestamp: "2026-05-26T11:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:46:03Z" name: sequence-graph-5b0de.18b31affa7e452b2 namespace: kserve-ci-e2e-test resourceVersion: "18102" uid: 356495ea-bf00-4bf8-920d-022bee64f280 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:46:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-5b0de namespace: kserve-ci-e2e-test resourceVersion: "18083" uid: 04a228c4-ead0-4446-b1cb-a76088bd76e6 kind: Event lastTimestamp: "2026-05-26T11:46:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:33Z" name: sequence-graph-5b0de.18b31b0320589bb1 namespace: kserve-ci-e2e-test resourceVersion: "18331" uid: e3731d41-ab4c-4a1d-9de1-2ad3a6b5fc25 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:46:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-5b0de namespace: kserve-ci-e2e-test resourceVersion: "18083" uid: 04a228c4-ead0-4446-b1cb-a76088bd76e6 kind: Event lastTimestamp: "2026-05-26T11:46:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:46:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:33Z" name: sequence-graph-5b0de.18b31b03209e43ae namespace: kserve-ci-e2e-test resourceVersion: "18332" uid: ffbb11f8-6cce-4aef-b8c9-1eee3d42f272 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:55:17.946322Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fbdd2-58d5d54c6b-lps2t namespace: kserve-ci-e2e-test resourceVersion: "22511" uid: 7d6808ed-f916-452d-a0bc-bd782dda39a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-fbdd2-58d5d54c6b-lps2t to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:55:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:55:17Z" name: sequence-graph-fbdd2-58d5d54c6b-lps2t.18b31b80d8ab185e namespace: kserve-ci-e2e-test resourceVersion: "22516" uid: 3502a091-42a6-43a7-b931-68377715b960 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:18Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fbdd2-58d5d54c6b-lps2t namespace: kserve-ci-e2e-test resourceVersion: "22513" uid: 7d6808ed-f916-452d-a0bc-bd782dda39a1 kind: Event lastTimestamp: "2026-05-26T11:55:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-fbdd2-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:18Z" name: sequence-graph-fbdd2-58d5d54c6b-lps2t.18b31b80e60b8afc namespace: kserve-ci-e2e-test resourceVersion: "22533" uid: 7db2c6eb-316f-44fe-9efc-01d73c78d040 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:19Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fbdd2-58d5d54c6b-lps2t namespace: kserve-ci-e2e-test resourceVersion: "22521" uid: 7d6808ed-f916-452d-a0bc-bd782dda39a1 kind: Event lastTimestamp: "2026-05-26T11:55:19Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:55:19Z" name: sequence-graph-fbdd2-58d5d54c6b-lps2t.18b31b811795ab95 namespace: kserve-ci-e2e-test resourceVersion: "22544" uid: 86b6468d-d7f8-4186-932f-19fd359796b2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fbdd2} kind: Pod name: sequence-graph-fbdd2-58d5d54c6b-lps2t namespace: kserve-ci-e2e-test resourceVersion: "22513" uid: 7d6808ed-f916-452d-a0bc-bd782dda39a1 kind: Event lastTimestamp: "2026-05-26T11:55:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-26T11:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:19Z" name: sequence-graph-fbdd2-58d5d54c6b-lps2t.18b31b8118de927a namespace: kserve-ci-e2e-test resourceVersion: "22546" uid: 8ea4f0f8-ca9f-48d4-bf05-243e7afa23dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fbdd2} kind: Pod name: sequence-graph-fbdd2-58d5d54c6b-lps2t namespace: kserve-ci-e2e-test resourceVersion: "22513" uid: 7d6808ed-f916-452d-a0bc-bd782dda39a1 kind: Event lastTimestamp: "2026-05-26T11:55:19Z" message: 'Created container: sequence-graph-fbdd2' metadata: creationTimestamp: "2026-05-26T11:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:19Z" name: sequence-graph-fbdd2-58d5d54c6b-lps2t.18b31b811c5084b9 namespace: kserve-ci-e2e-test resourceVersion: "22547" uid: 993073c7-97f6-4b14-9a95-6d915a56044c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fbdd2} kind: Pod name: sequence-graph-fbdd2-58d5d54c6b-lps2t namespace: kserve-ci-e2e-test resourceVersion: "22513" uid: 7d6808ed-f916-452d-a0bc-bd782dda39a1 kind: Event lastTimestamp: "2026-05-26T11:55:19Z" message: Started container sequence-graph-fbdd2 metadata: creationTimestamp: "2026-05-26T11:55:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:19Z" name: sequence-graph-fbdd2-58d5d54c6b-lps2t.18b31b811ccb2f90 namespace: kserve-ci-e2e-test resourceVersion: "22548" uid: c4e9d4ad-3529-43c2-9c8f-ef71ff170dae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fbdd2} kind: Pod name: sequence-graph-fbdd2-58d5d54c6b-lps2t namespace: kserve-ci-e2e-test resourceVersion: "22513" uid: 7d6808ed-f916-452d-a0bc-bd782dda39a1 kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: Stopping container sequence-graph-fbdd2 metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:28Z" name: sequence-graph-fbdd2-58d5d54c6b-lps2t.18b31b833447d74a namespace: kserve-ci-e2e-test resourceVersion: "22642" uid: 65aed64a-e849-4210-97c7-74f2f45dfb4f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:55:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fbdd2} kind: Pod name: sequence-graph-fbdd2-58d5d54c6b-lps2t namespace: kserve-ci-e2e-test resourceVersion: "22513" uid: 7d6808ed-f916-452d-a0bc-bd782dda39a1 kind: Event lastTimestamp: "2026-05-26T11:55:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T11:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:55Z" name: sequence-graph-fbdd2-58d5d54c6b-lps2t.18b31b83d6defd41 namespace: kserve-ci-e2e-test resourceVersion: "23003" uid: edf90cfc-eaa1-4325-9f17-742d84081468 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-fbdd2-58d5d54c6b namespace: kserve-ci-e2e-test resourceVersion: "22508" uid: f8e93391-e581-4371-b076-8b15ac7ded56 kind: Event lastTimestamp: "2026-05-26T11:55:17Z" message: 'Created pod: sequence-graph-fbdd2-58d5d54c6b-lps2t' metadata: creationTimestamp: "2026-05-26T11:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:17Z" name: sequence-graph-fbdd2-58d5d54c6b.18b31b80d8429998 namespace: kserve-ci-e2e-test resourceVersion: "22512" uid: 91b7c333-00a2-4e97-9859-4825b72620d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-fbdd2 namespace: kserve-ci-e2e-test resourceVersion: "22506" uid: c881ab37-452b-42f9-aa73-b7eaf9b39827 kind: Event lastTimestamp: "2026-05-26T11:55:17Z" message: Scaled up replica set sequence-graph-fbdd2-58d5d54c6b from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:17Z" name: sequence-graph-fbdd2.18b31b80d7a8d94b namespace: kserve-ci-e2e-test resourceVersion: "22509" uid: 1c3a4c33-36ba-45d2-85e3-1c01e7d9fcfb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-fbdd2 namespace: kserve-ci-e2e-test resourceVersion: "22524" uid: d22f8970-a49c-4151-a3b2-ee2960b884d7 kind: Event lastTimestamp: "2026-05-26T11:55:18Z" message: InferenceGraph [sequence-graph-fbdd2] is Ready metadata: creationTimestamp: "2026-05-26T11:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:18Z" name: sequence-graph-fbdd2.18b31b80dc9ba2c8 namespace: kserve-ci-e2e-test resourceVersion: "22526" uid: 582df3c0-dbad-4a3a-928c-dd46a690acc1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T12:06:23.738933Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-52624-856bdd7d4b-swwrx namespace: kserve-ci-e2e-test resourceVersion: "28467" uid: bb9ca510-dcf7-4b6a-9065-b2abb57680ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-52624-856bdd7d4b-swwrx to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T12:06:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T12:06:23Z" name: splitter-graph-52624-856bdd7d4b-swwrx.18b31c1bdd00191c namespace: kserve-ci-e2e-test resourceVersion: "28474" uid: 8859c73a-b0ad-46d6-9405-3a00fa8423b8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:06:23Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-52624-856bdd7d4b-swwrx namespace: kserve-ci-e2e-test resourceVersion: "28471" uid: bb9ca510-dcf7-4b6a-9065-b2abb57680ff kind: Event lastTimestamp: "2026-05-26T12:06:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-52624-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T12:06:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:06:23Z" name: splitter-graph-52624-856bdd7d4b-swwrx.18b31c1be6dbe44a namespace: kserve-ci-e2e-test resourceVersion: "28488" uid: cc74d14d-6bb2-4997-b162-f70b1d85adc2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:06:24Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-52624-856bdd7d4b-swwrx namespace: kserve-ci-e2e-test resourceVersion: "28478" uid: bb9ca510-dcf7-4b6a-9065-b2abb57680ff kind: Event lastTimestamp: "2026-05-26T12:06:24Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T12:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T12:06:24Z" name: splitter-graph-52624-856bdd7d4b-swwrx.18b31c1c1ab83f51 namespace: kserve-ci-e2e-test resourceVersion: "28497" uid: a584a94d-80c9-4abe-b25a-086176d98ed6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:06:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-52624} kind: Pod name: splitter-graph-52624-856bdd7d4b-swwrx namespace: kserve-ci-e2e-test resourceVersion: "28471" uid: bb9ca510-dcf7-4b6a-9065-b2abb57680ff kind: Event lastTimestamp: "2026-05-26T12:06:24Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-26T12:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:06:24Z" name: splitter-graph-52624-856bdd7d4b-swwrx.18b31c1c1c3f4301 namespace: kserve-ci-e2e-test resourceVersion: "28499" uid: 41729865-8b81-4503-9dc9-e05355d117c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:06:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-52624} kind: Pod name: splitter-graph-52624-856bdd7d4b-swwrx namespace: kserve-ci-e2e-test resourceVersion: "28471" uid: bb9ca510-dcf7-4b6a-9065-b2abb57680ff kind: Event lastTimestamp: "2026-05-26T12:06:24Z" message: 'Created container: splitter-graph-52624' metadata: creationTimestamp: "2026-05-26T12:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:06:24Z" name: splitter-graph-52624-856bdd7d4b-swwrx.18b31c1c1f7bbdd5 namespace: kserve-ci-e2e-test resourceVersion: "28501" uid: 6d4c04f6-26c2-4043-8a4e-43588590636d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:06:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-52624} kind: Pod name: splitter-graph-52624-856bdd7d4b-swwrx namespace: kserve-ci-e2e-test resourceVersion: "28471" uid: bb9ca510-dcf7-4b6a-9065-b2abb57680ff kind: Event lastTimestamp: "2026-05-26T12:06:24Z" message: Started container splitter-graph-52624 metadata: creationTimestamp: "2026-05-26T12:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:06:24Z" name: splitter-graph-52624-856bdd7d4b-swwrx.18b31c1c1ffab2dd namespace: kserve-ci-e2e-test resourceVersion: "28502" uid: 6ddc396d-e5e1-466f-904d-3d4586d7a4e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-52624} kind: Pod name: splitter-graph-52624-856bdd7d4b-swwrx namespace: kserve-ci-e2e-test resourceVersion: "28471" uid: bb9ca510-dcf7-4b6a-9065-b2abb57680ff kind: Event lastTimestamp: "2026-05-26T12:14:38Z" message: Stopping container splitter-graph-52624 metadata: creationTimestamp: "2026-05-26T12:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:14:38Z" name: splitter-graph-52624-856bdd7d4b-swwrx.18b31c8f0adce129 namespace: kserve-ci-e2e-test resourceVersion: "31887" uid: 419a86d5-379e-49b7-bc59-926080839f3c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T12:14:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-52624} kind: Pod name: splitter-graph-52624-856bdd7d4b-swwrx namespace: kserve-ci-e2e-test resourceVersion: "28471" uid: bb9ca510-dcf7-4b6a-9065-b2abb57680ff kind: Event lastTimestamp: "2026-05-26T12:15:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T12:14:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:15:06Z" name: splitter-graph-52624-856bdd7d4b-swwrx.18b31c8fd170e370 namespace: kserve-ci-e2e-test resourceVersion: "32127" uid: ea612f9a-76f8-4138-8583-8d1ddfc4a294 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:06:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-52624-856bdd7d4b namespace: kserve-ci-e2e-test resourceVersion: "28464" uid: 2529298a-7c1c-492b-9d61-9331d4586783 kind: Event lastTimestamp: "2026-05-26T12:06:23Z" message: 'Created pod: splitter-graph-52624-856bdd7d4b-swwrx' metadata: creationTimestamp: "2026-05-26T12:06:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:06:23Z" name: splitter-graph-52624-856bdd7d4b.18b31c1bdc5d15f0 namespace: kserve-ci-e2e-test resourceVersion: "28469" uid: 139fe102-4e63-4655-bf80-01c1155ab7b0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:06:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-52624 namespace: kserve-ci-e2e-test resourceVersion: "28462" uid: aaf8e333-b1e0-4568-9b27-c9abc2595fb7 kind: Event lastTimestamp: "2026-05-26T12:06:23Z" message: Scaled up replica set splitter-graph-52624-856bdd7d4b from 0 to 1 metadata: creationTimestamp: "2026-05-26T12:06:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:06:23Z" name: splitter-graph-52624.18b31c1bdb9b0254 namespace: kserve-ci-e2e-test resourceVersion: "28466" uid: d1bf9ac8-3448-42b8-b8d0-368b97207c79 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:06:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-52624 namespace: kserve-ci-e2e-test resourceVersion: "28482" uid: e309a52f-be9a-459c-a01a-cafbc6fbfa15 kind: Event lastTimestamp: "2026-05-26T12:06:23Z" message: InferenceGraph [splitter-graph-52624] is Ready metadata: creationTimestamp: "2026-05-26T12:06:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:06:23Z" name: splitter-graph-52624.18b31c1be10f5cc4 namespace: kserve-ci-e2e-test resourceVersion: "28485" uid: aa00391a-0d5d-416e-a9af-8379c9987778 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:06:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-52624 namespace: kserve-ci-e2e-test resourceVersion: "28463" uid: 9d652019-cb71-4884-a530-c9e5d615812d kind: Event lastTimestamp: "2026-05-26T12:06:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:06:53Z" name: splitter-graph-52624.18b31c1f5a566070 namespace: kserve-ci-e2e-test resourceVersion: "28713" uid: d052b45d-4377-417a-a5b2-d7780ed27d3a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:06:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-52624 namespace: kserve-ci-e2e-test resourceVersion: "28463" uid: 9d652019-cb71-4884-a530-c9e5d615812d kind: Event lastTimestamp: "2026-05-26T12:06:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:06:53Z" name: splitter-graph-52624.18b31c1f5ac1fe71 namespace: kserve-ci-e2e-test resourceVersion: "28714" uid: 6c9c712e-20df-4660-9792-3d17530c5043 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T12:05:13.460130Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-81833-78975b54d7-2xl89 namespace: kserve-ci-e2e-test resourceVersion: "27653" uid: 214626d8-620f-4fb6-ae8b-d24a040dac22 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-81833-78975b54d7-2xl89 to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T12:05:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T12:05:13Z" name: splitter-graph-81833-78975b54d7-2xl89.18b31c0b800ed5f2 namespace: kserve-ci-e2e-test resourceVersion: "27658" uid: 39383978-a8bb-40d8-ba33-ed9d39bbcb37 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:13Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-81833-78975b54d7-2xl89 namespace: kserve-ci-e2e-test resourceVersion: "27661" uid: 214626d8-620f-4fb6-ae8b-d24a040dac22 kind: Event lastTimestamp: "2026-05-26T12:05:13Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T12:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T12:05:13Z" name: splitter-graph-81833-78975b54d7-2xl89.18b31c0b9a0ee596 namespace: kserve-ci-e2e-test resourceVersion: "27679" uid: 71a49f63-48a7-469a-842c-e93d453ea61a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-81833} kind: Pod name: splitter-graph-81833-78975b54d7-2xl89 namespace: kserve-ci-e2e-test resourceVersion: "27656" uid: 214626d8-620f-4fb6-ae8b-d24a040dac22 kind: Event lastTimestamp: "2026-05-26T12:05:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-26T12:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:13Z" name: splitter-graph-81833-78975b54d7-2xl89.18b31c0b9b5e02da namespace: kserve-ci-e2e-test resourceVersion: "27681" uid: 02422d36-7918-4a8f-a2df-483c943bb729 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-81833} kind: Pod name: splitter-graph-81833-78975b54d7-2xl89 namespace: kserve-ci-e2e-test resourceVersion: "27656" uid: 214626d8-620f-4fb6-ae8b-d24a040dac22 kind: Event lastTimestamp: "2026-05-26T12:05:13Z" message: 'Created container: splitter-graph-81833' metadata: creationTimestamp: "2026-05-26T12:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:13Z" name: splitter-graph-81833-78975b54d7-2xl89.18b31c0b9f494c10 namespace: kserve-ci-e2e-test resourceVersion: "27682" uid: 71c686a9-3688-447f-854c-efe3b8046a00 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-81833} kind: Pod name: splitter-graph-81833-78975b54d7-2xl89 namespace: kserve-ci-e2e-test resourceVersion: "27656" uid: 214626d8-620f-4fb6-ae8b-d24a040dac22 kind: Event lastTimestamp: "2026-05-26T12:05:13Z" message: Started container splitter-graph-81833 metadata: creationTimestamp: "2026-05-26T12:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:13Z" name: splitter-graph-81833-78975b54d7-2xl89.18b31c0b9fbde3ac namespace: kserve-ci-e2e-test resourceVersion: "27683" uid: 83a725a0-3782-4e69-94f3-359d70ffdeb7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-81833} kind: Pod name: splitter-graph-81833-78975b54d7-2xl89 namespace: kserve-ci-e2e-test resourceVersion: "27656" uid: 214626d8-620f-4fb6-ae8b-d24a040dac22 kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: Stopping container splitter-graph-81833 metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:23Z" name: splitter-graph-81833-78975b54d7-2xl89.18b31c0dd877667d namespace: kserve-ci-e2e-test resourceVersion: "27778" uid: 8f73a794-f0dc-4b4f-ac72-310a97ae6813 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T12:05:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-81833} kind: Pod name: splitter-graph-81833-78975b54d7-2xl89 namespace: kserve-ci-e2e-test resourceVersion: "27656" uid: 214626d8-620f-4fb6-ae8b-d24a040dac22 kind: Event lastTimestamp: "2026-05-26T12:05:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T12:05:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:50Z" name: splitter-graph-81833-78975b54d7-2xl89.18b31c0e5015f3ac namespace: kserve-ci-e2e-test resourceVersion: "28183" uid: 45547fc7-6e06-4a8b-8c29-94d83e52f986 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-81833-78975b54d7 namespace: kserve-ci-e2e-test resourceVersion: "27650" uid: fc5c2f89-adfa-4e2c-9524-52b7697374dc kind: Event lastTimestamp: "2026-05-26T12:05:13Z" message: 'Created pod: splitter-graph-81833-78975b54d7-2xl89' metadata: creationTimestamp: "2026-05-26T12:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:13Z" name: splitter-graph-81833-78975b54d7.18b31c0b7f8d07f9 namespace: kserve-ci-e2e-test resourceVersion: "27655" uid: f032aacf-14d5-47de-83a3-c0e3de3eda32 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-81833 namespace: kserve-ci-e2e-test resourceVersion: "27648" uid: d50675ee-ecdb-4bb4-be3f-c5da664c5992 kind: Event lastTimestamp: "2026-05-26T12:05:13Z" message: Scaled up replica set splitter-graph-81833-78975b54d7 from 0 to 1 metadata: creationTimestamp: "2026-05-26T12:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:13Z" name: splitter-graph-81833.18b31c0b7e960981 namespace: kserve-ci-e2e-test resourceVersion: "27651" uid: 7c155c34-668f-4385-b88d-4aa8c2e67664 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-81833 namespace: kserve-ci-e2e-test resourceVersion: "27667" uid: 31a3592c-f56b-41b9-8f52-ef322c81352d kind: Event lastTimestamp: "2026-05-26T12:05:13Z" message: InferenceGraph [splitter-graph-81833] is Ready metadata: creationTimestamp: "2026-05-26T12:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:13Z" name: splitter-graph-81833.18b31c0b843b6696 namespace: kserve-ci-e2e-test resourceVersion: "27670" uid: 330d3eb3-21e7-483e-82f2-973cd1cb83fd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-81833 namespace: kserve-ci-e2e-test resourceVersion: "27639" uid: 31a3592c-f56b-41b9-8f52-ef322c81352d kind: Event lastTimestamp: "2026-05-26T12:05:13Z" message: 'Failed to update status for InferenceGraph "splitter-graph-81833": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-81833": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:13Z" name: splitter-graph-81833.18b31c0b86e59f57 namespace: kserve-ci-e2e-test resourceVersion: "27673" uid: 0fb3c092-25af-432e-b95f-696a565b3446 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-81833 namespace: kserve-ci-e2e-test resourceVersion: "27639" uid: 31a3592c-f56b-41b9-8f52-ef322c81352d kind: Event lastTimestamp: "2026-05-26T12:05:13Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "splitter-graph-81833": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:05:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:13Z" name: splitter-graph-81833.18b31c0b86e5fc27 namespace: kserve-ci-e2e-test resourceVersion: "27674" uid: 1abb7e2b-e2b0-4caf-8a23-e60e492c75a5 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:54:58.387638Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0a5df-predictor-54955c646-lbvtd namespace: kserve-ci-e2e-test resourceVersion: "22245" uid: 02c39fc2-6c59-46a7-b92c-0fa902ede916 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0a5df-predictor-54955c646-lbvtd to ip-10-0-135-71.ec2.internal metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:54:58Z" name: success-200-isvc-0a5df-predictor-54955c646-lbvtd.18b31b7c4ae139d9 namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: db8fcea5-28ca-41ca-8b20-fe77e11d31be reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0a5df-predictor-54955c646-lbvtd namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 02c39fc2-6c59-46a7-b92c-0fa902ede916 kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-0a5df-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:58Z" name: success-200-isvc-0a5df-predictor-54955c646-lbvtd.18b31b7c582efaf3 namespace: kserve-ci-e2e-test resourceVersion: "22284" uid: 191d81fd-0753-4c8b-8f5d-0f5bc757ad89 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0a5df-predictor-54955c646-lbvtd namespace: kserve-ci-e2e-test resourceVersion: "22256" uid: 02c39fc2-6c59-46a7-b92c-0fa902ede916 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:54:59Z" name: success-200-isvc-0a5df-predictor-54955c646-lbvtd.18b31b7c88641785 namespace: kserve-ci-e2e-test resourceVersion: "22299" uid: 7c90efe9-5371-4333-a8a7-c99c365d375f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a5df-predictor-54955c646-lbvtd namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 02c39fc2-6c59-46a7-b92c-0fa902ede916 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:59Z" name: success-200-isvc-0a5df-predictor-54955c646-lbvtd.18b31b7c89bac9dc namespace: kserve-ci-e2e-test resourceVersion: "22301" uid: 5e37238b-58a9-44d5-b409-e8a129f5ccfd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a5df-predictor-54955c646-lbvtd namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 02c39fc2-6c59-46a7-b92c-0fa902ede916 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:59Z" name: success-200-isvc-0a5df-predictor-54955c646-lbvtd.18b31b7c8d4935ff namespace: kserve-ci-e2e-test resourceVersion: "22303" uid: dad61f55-19fb-48fc-8b13-b7a8c59716dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a5df-predictor-54955c646-lbvtd namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 02c39fc2-6c59-46a7-b92c-0fa902ede916 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:59Z" name: success-200-isvc-0a5df-predictor-54955c646-lbvtd.18b31b7c8dbf938d namespace: kserve-ci-e2e-test resourceVersion: "22304" uid: 4af8ca10-b801-405b-b504-ef3265491538 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a5df-predictor-54955c646-lbvtd namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 02c39fc2-6c59-46a7-b92c-0fa902ede916 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:59Z" name: success-200-isvc-0a5df-predictor-54955c646-lbvtd.18b31b7c8dcb1470 namespace: kserve-ci-e2e-test resourceVersion: "22305" uid: 990da986-4eb1-4623-aea2-c33cb33ed637 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a5df-predictor-54955c646-lbvtd namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 02c39fc2-6c59-46a7-b92c-0fa902ede916 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:59Z" name: success-200-isvc-0a5df-predictor-54955c646-lbvtd.18b31b7c914c5914 namespace: kserve-ci-e2e-test resourceVersion: "22309" uid: 50255cd4-8ac4-4a2d-b367-06fcc8b8038d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a5df-predictor-54955c646-lbvtd namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 02c39fc2-6c59-46a7-b92c-0fa902ede916 kind: Event lastTimestamp: "2026-05-26T11:54:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:54:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:59Z" name: success-200-isvc-0a5df-predictor-54955c646-lbvtd.18b31b7c91c408f1 namespace: kserve-ci-e2e-test resourceVersion: "22310" uid: f4e886e7-fa36-4cea-8f91-de65b61cd2f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:55:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a5df-predictor-54955c646-lbvtd namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 02c39fc2-6c59-46a7-b92c-0fa902ede916 kind: Event lastTimestamp: "2026-05-26T11:55:36Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:55:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:36Z" name: success-200-isvc-0a5df-predictor-54955c646-lbvtd.18b31b7cd7f9f5ac namespace: kserve-ci-e2e-test resourceVersion: "22841" uid: 3ba9abed-b155-400d-959b-6dfdea94b07a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0a5df-predictor-54955c646-lbvtd namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 02c39fc2-6c59-46a7-b92c-0fa902ede916 kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:13Z" name: success-200-isvc-0a5df-predictor-54955c646-lbvtd.18b31bfd7efb4f45 namespace: kserve-ci-e2e-test resourceVersion: "26895" uid: 7b7efe26-a4db-4a57-8af5-9ca643cb16cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0a5df-predictor-54955c646-lbvtd namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: 02c39fc2-6c59-46a7-b92c-0fa902ede916 kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:13Z" name: success-200-isvc-0a5df-predictor-54955c646-lbvtd.18b31bfd7efbbd21 namespace: kserve-ci-e2e-test resourceVersion: "26896" uid: dca2e3f9-44fe-4173-97bd-e22464e9100e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0a5df-predictor-54955c646 namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 6678b18b-1c55-4614-915a-c31383dde09f kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: 'Created pod: success-200-isvc-0a5df-predictor-54955c646-lbvtd' metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:58Z" name: success-200-isvc-0a5df-predictor-54955c646.18b31b7c4a4fdbff namespace: kserve-ci-e2e-test resourceVersion: "22247" uid: 7a676496-36b2-410f-a350-713e692b5171 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0a5df-predictor namespace: kserve-ci-e2e-test resourceVersion: "22225" uid: 588470ee-cb9a-47a2-b9a3-b2eded267cba kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: Scaled up replica set success-200-isvc-0a5df-predictor-54955c646 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:58Z" name: success-200-isvc-0a5df-predictor.18b31b7c487f476d namespace: kserve-ci-e2e-test resourceVersion: "22236" uid: 00b0d819-75f1-4265-9148-a6b38264b9d1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:55:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0a5df-predictor namespace: kserve-ci-e2e-test resourceVersion: "22229" uid: e14319f5-c722-41c5-bc2e-c386f039de44 kind: Event lastTimestamp: "2026-05-26T11:55:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:43Z" name: success-200-isvc-0a5df-predictor.18b31b7fc73a3df7 namespace: kserve-ci-e2e-test resourceVersion: "22895" uid: fd25dcc5-4820-463a-b324-0b7c96722a4b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:55:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0a5df-predictor namespace: kserve-ci-e2e-test resourceVersion: "22229" uid: e14319f5-c722-41c5-bc2e-c386f039de44 kind: Event lastTimestamp: "2026-05-26T11:55:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:43Z" name: success-200-isvc-0a5df-predictor.18b31b7fc7a26bb9 namespace: kserve-ci-e2e-test resourceVersion: "22896" uid: aafd1284-0035-4ce8-a950-114e97346337 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:55:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0a5df-predictor namespace: kserve-ci-e2e-test resourceVersion: "22458" uid: e14319f5-c722-41c5-bc2e-c386f039de44 kind: Event lastTimestamp: "2026-05-26T11:56:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:13Z" name: success-200-isvc-0a5df-predictor.18b31b8a43daf39d namespace: kserve-ci-e2e-test resourceVersion: "23187" uid: 0f6f65c5-25bb-46e4-96ed-9d9ed02cb572 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:55:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0a5df-predictor namespace: kserve-ci-e2e-test resourceVersion: "22458" uid: e14319f5-c722-41c5-bc2e-c386f039de44 kind: Event lastTimestamp: "2026-05-26T11:56:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:13Z" name: success-200-isvc-0a5df-predictor.18b31b8a4447fbe6 namespace: kserve-ci-e2e-test resourceVersion: "23188" uid: 289d8222-935b-45f7-9af6-374899da0f68 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0a5df namespace: kserve-ci-e2e-test resourceVersion: "22200" uid: 4afcf45e-42c9-4e0d-b19f-70fd46eaf8fc kind: Event lastTimestamp: "2026-05-26T11:55:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:46Z" name: success-200-isvc-0a5df.18b31b7c48801f7c namespace: kserve-ci-e2e-test resourceVersion: "22941" uid: 40446a3a-d696-416e-8b62-c914087a1dca reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0a5df namespace: kserve-ci-e2e-test resourceVersion: "22200" uid: 4afcf45e-42c9-4e0d-b19f-70fd46eaf8fc kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: 'Failed to update status for InferenceService "success-200-isvc-0a5df": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0a5df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:58Z" name: success-200-isvc-0a5df.18b31b7c49246b85 namespace: kserve-ci-e2e-test resourceVersion: "22241" uid: f737755f-74ad-4782-8f13-38a3c50e028f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0a5df namespace: kserve-ci-e2e-test resourceVersion: "22200" uid: 4afcf45e-42c9-4e0d-b19f-70fd46eaf8fc kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-0a5df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:58Z" name: success-200-isvc-0a5df.18b31b7c4924ac53 namespace: kserve-ci-e2e-test resourceVersion: "22244" uid: 255f2f2f-529d-47f1-8427-bd7389d9e808 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0a5df namespace: kserve-ci-e2e-test resourceVersion: "22934" uid: 4afcf45e-42c9-4e0d-b19f-70fd46eaf8fc kind: Event lastTimestamp: "2026-05-26T11:55:46Z" message: InferenceService [success-200-isvc-0a5df] is Ready metadata: creationTimestamp: "2026-05-26T11:55:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:46Z" name: success-200-isvc-0a5df.18b31b87928dccf5 namespace: kserve-ci-e2e-test resourceVersion: "22936" uid: 377d1768-18bc-4935-a9ca-210d79a50a0a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T12:04:42.935054Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks namespace: kserve-ci-e2e-test resourceVersion: "27297" uid: 6fc22dc1-5779-4102-9260-8107f828eddb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-152bc-predictor-7456d7b59b-hxqks to ip-10-0-135-71.ec2.internal metadata: creationTimestamp: "2026-05-26T12:04:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T12:04:42Z" name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks.18b31c04649ef6c7 namespace: kserve-ci-e2e-test resourceVersion: "27301" uid: 03679f39-446d-4cad-90c6-2ccaec2f6763 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 6fc22dc1-5779-4102-9260-8107f828eddb kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-152bc-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:43Z" name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks.18b31c0471e0abf4 namespace: kserve-ci-e2e-test resourceVersion: "27336" uid: 0e49a0b4-e0f6-4707-8c7d-c66027c2a17e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks namespace: kserve-ci-e2e-test resourceVersion: "27306" uid: 6fc22dc1-5779-4102-9260-8107f828eddb kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T12:04:43Z" name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks.18b31c04a2484545 namespace: kserve-ci-e2e-test resourceVersion: "27361" uid: 822b2c6e-ce5f-441f-a92f-64066f5ed9fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 6fc22dc1-5779-4102-9260-8107f828eddb kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T12:04:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:43Z" name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks.18b31c04a3c1d1c4 namespace: kserve-ci-e2e-test resourceVersion: "27363" uid: eb1d907c-fc01-44f2-930a-0b72be17b5f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 6fc22dc1-5779-4102-9260-8107f828eddb kind: Event lastTimestamp: "2026-05-26T12:04:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T12:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:44Z" name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks.18b31c04a703bcc8 namespace: kserve-ci-e2e-test resourceVersion: "27365" uid: d7612acd-8ed2-4280-a795-f9774c4f08b3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 6fc22dc1-5779-4102-9260-8107f828eddb kind: Event lastTimestamp: "2026-05-26T12:04:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T12:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:44Z" name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks.18b31c04a774aed8 namespace: kserve-ci-e2e-test resourceVersion: "27367" uid: f39bc51e-0433-4eb7-be18-cbd1d3b21e7d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 6fc22dc1-5779-4102-9260-8107f828eddb kind: Event lastTimestamp: "2026-05-26T12:04:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T12:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:44Z" name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks.18b31c04a780623f namespace: kserve-ci-e2e-test resourceVersion: "27368" uid: 84724ef3-a836-42af-a247-b754f0c45bcb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 6fc22dc1-5779-4102-9260-8107f828eddb kind: Event lastTimestamp: "2026-05-26T12:04:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T12:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:44Z" name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks.18b31c04aa591b8b namespace: kserve-ci-e2e-test resourceVersion: "27370" uid: 782acc2c-fb4e-427b-9767-b59b3ecc3d93 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 6fc22dc1-5779-4102-9260-8107f828eddb kind: Event lastTimestamp: "2026-05-26T12:04:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:44Z" name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks.18b31c04aac89ec1 namespace: kserve-ci-e2e-test resourceVersion: "27372" uid: 24f47603-9c9f-4c90-887a-cb24e8c7f16c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T12:04:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 6fc22dc1-5779-4102-9260-8107f828eddb kind: Event lastTimestamp: "2026-05-26T12:05:20Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T12:04:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:20Z" name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks.18b31c04d5b3cea0 namespace: kserve-ci-e2e-test resourceVersion: "27739" uid: 42caba28-e52f-4311-950e-5204d5e843e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 6fc22dc1-5779-4102-9260-8107f828eddb kind: Event lastTimestamp: "2026-05-26T12:22:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T12:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:22:02Z" name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks.18b31cf67261f992 namespace: kserve-ci-e2e-test resourceVersion: "34982" uid: 73dfa718-0cba-4089-b1ba-bdd30338222b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks namespace: kserve-ci-e2e-test resourceVersion: "27299" uid: 6fc22dc1-5779-4102-9260-8107f828eddb kind: Event lastTimestamp: "2026-05-26T12:22:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:22:02Z" name: success-200-isvc-152bc-predictor-7456d7b59b-hxqks.18b31cf67262ac44 namespace: kserve-ci-e2e-test resourceVersion: "34983" uid: 0d81299e-c1c8-4666-95a3-e4fe92ac1d53 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-152bc-predictor-7456d7b59b namespace: kserve-ci-e2e-test resourceVersion: "27283" uid: 3b0ff5af-e991-44ed-a6ca-9095055b621f kind: Event lastTimestamp: "2026-05-26T12:04:42Z" message: 'Created pod: success-200-isvc-152bc-predictor-7456d7b59b-hxqks' metadata: creationTimestamp: "2026-05-26T12:04:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:04:42Z" name: success-200-isvc-152bc-predictor-7456d7b59b.18b31c04642b2094 namespace: kserve-ci-e2e-test resourceVersion: "27298" uid: a9c8be93-30bd-4c55-ac64-89bf51fcc2aa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-152bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "27278" uid: 5abd6ecd-9eea-4d91-b162-12a4595cbda1 kind: Event lastTimestamp: "2026-05-26T12:04:42Z" message: Scaled up replica set success-200-isvc-152bc-predictor-7456d7b59b from 0 to 1 metadata: creationTimestamp: "2026-05-26T12:04:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:04:42Z" name: success-200-isvc-152bc-predictor.18b31c04623262b2 namespace: kserve-ci-e2e-test resourceVersion: "27288" uid: a945d237-e4d0-4501-86c4-046a245808ba reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:04:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-152bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "27282" uid: a9104b73-b178-4ddc-be0d-404f81e959ca kind: Event lastTimestamp: "2026-05-26T12:05:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:04:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:12Z" name: success-200-isvc-152bc-predictor.18b31c07e28ab7c0 namespace: kserve-ci-e2e-test resourceVersion: "27633" uid: 79b62183-b5ec-47b8-b315-f19caed59128 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:04:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-152bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "27282" uid: a9104b73-b178-4ddc-be0d-404f81e959ca kind: Event lastTimestamp: "2026-05-26T12:05:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:04:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:12Z" name: success-200-isvc-152bc-predictor.18b31c07e3523e6e namespace: kserve-ci-e2e-test resourceVersion: "27634" uid: 91adbbb6-dcc2-4957-bcfb-66903de5e8a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:05:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-152bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "27490" uid: a9104b73-b178-4ddc-be0d-404f81e959ca kind: Event lastTimestamp: "2026-05-26T12:05:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:42Z" name: success-200-isvc-152bc-predictor.18b31c0ee14a1856 namespace: kserve-ci-e2e-test resourceVersion: "28103" uid: 04035ca2-54e6-458a-b70b-2d07c80212cb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:05:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-152bc-predictor namespace: kserve-ci-e2e-test resourceVersion: "27490" uid: a9104b73-b178-4ddc-be0d-404f81e959ca kind: Event lastTimestamp: "2026-05-26T12:05:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:43Z" name: success-200-isvc-152bc-predictor.18b31c0ee1aa4588 namespace: kserve-ci-e2e-test resourceVersion: "28104" uid: 26e6b6f7-ab89-4be9-ad0e-8b7afe57b002 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T12:04:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-152bc namespace: kserve-ci-e2e-test resourceVersion: "27249" uid: a3d8e0d3-7024-4210-9c85-3c53366f2984 kind: Event lastTimestamp: "2026-05-26T12:05:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T12:04:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:30Z" name: success-200-isvc-152bc.18b31c0462267f9e namespace: kserve-ci-e2e-test resourceVersion: "27977" uid: e5cd18cd-4c7a-4513-a28c-0b00c3a6dbd5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-152bc namespace: kserve-ci-e2e-test resourceVersion: "27249" uid: a3d8e0d3-7024-4210-9c85-3c53366f2984 kind: Event lastTimestamp: "2026-05-26T12:04:42Z" message: 'Failed to update status for InferenceService "success-200-isvc-152bc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-152bc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:04:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:04:42Z" name: success-200-isvc-152bc.18b31c0462ac5bf0 namespace: kserve-ci-e2e-test resourceVersion: "27291" uid: a091f754-596b-4131-a438-2af39c9500c7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-152bc namespace: kserve-ci-e2e-test resourceVersion: "27249" uid: a3d8e0d3-7024-4210-9c85-3c53366f2984 kind: Event lastTimestamp: "2026-05-26T12:04:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-152bc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:04:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:04:42Z" name: success-200-isvc-152bc.18b31c0462acb06b namespace: kserve-ci-e2e-test resourceVersion: "27292" uid: 4ecb773b-354b-4c13-8121-94c876cdf9a4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-152bc namespace: kserve-ci-e2e-test resourceVersion: "27974" uid: a3d8e0d3-7024-4210-9c85-3c53366f2984 kind: Event lastTimestamp: "2026-05-26T12:05:30Z" message: InferenceService [success-200-isvc-152bc] is Ready metadata: creationTimestamp: "2026-05-26T12:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:30Z" name: success-200-isvc-152bc.18b31c0f911613be namespace: kserve-ci-e2e-test resourceVersion: "27975" uid: e0ef69b9-780c-4709-ae1a-13e8b489a051 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:43:02.652512Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-378f2-predictor-8bbd48d54-vp5rd to ip-10-0-135-71.ec2.internal metadata: creationTimestamp: "2026-05-26T11:43:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:43:02Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31ad5a5bde90f namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: bfe1e2eb-3847-474c-a6dd-2949b4b09aa2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:43:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-378f2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:02Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31ad5b0d30239 namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: 68ddc75d-eb91-4c1a-9243-c7141c31e8e5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:43:03Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:43:03Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31ad5e3877e3c namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 3c5f1e6f-aef1-41c8-96ca-50b78d771d30 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:43:03Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" metadata: creationTimestamp: "2026-05-26T11:43:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:03Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31ad5e4fec5a1 namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: 933617b9-3fd5-4a18-ae2f-4a8eda6e6ab9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:43:15Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" in 11.884s (11.884s including waiting). Image size: 1335713906 bytes.' metadata: creationTimestamp: "2026-05-26T11:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:15Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31ad8a95a88a3 namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: ce531b69-2104-4e07-9f91-28774669f579 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:43:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:15Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31ad8ad4d2f15 namespace: kserve-ci-e2e-test resourceVersion: "16151" uid: 751dd241-a238-45e1-b7a0-c27cf428f0d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:43:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:15Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31ad8adbf9138 namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: 6579991c-2db6-4603-b450-9fe18bc6c82b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:43:15Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-05-26T11:43:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:15Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31ad8adcd9d2f namespace: kserve-ci-e2e-test resourceVersion: "16153" uid: e8550bbb-b334-4b48-8695-de485dae14a5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:43:17Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.155s (2.155s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-05-26T11:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:17Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31ad92e41f0bd namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: d238d978-c4e7-4569-a5f6-c2b9df90c06a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:43:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:17Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31ad931b069bd namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: e1f69136-3ee4-436c-831c-fde4824ef1f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:43:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:43:17Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31ad932167a35 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: 6cee1789-1ea8-4d8d-a801-31ab988af7de reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T11:43:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:44:34Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:43:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:34Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31ad9498e01c9 namespace: kserve-ci-e2e-test resourceVersion: "17068" uid: 67221f9f-d13a-4e65-9402-34da7309c940 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:33Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31aeaafaa9acc namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: 859d109e-f513-48ac-bda6-63638cdb2b05 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:33Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31aeaafab51f2 namespace: kserve-ci-e2e-test resourceVersion: "16961" uid: 2a0d450a-3f2f-4cb2-b4c9-8a8325fd4543 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 8c56111a-f26e-4266-bb10-86bdcdf8f2d7 kind: Event lastTimestamp: "2026-05-26T11:44:34Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:34Z" name: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd.18b31aeafbc47448 namespace: kserve-ci-e2e-test resourceVersion: "17067" uid: a5dd11f9-ec9d-4b3d-a7db-13730d3002c1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-378f2-predictor-8bbd48d54 namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: ea665287-d86d-401e-9f4c-b76269a4ae03 kind: Event lastTimestamp: "2026-05-26T11:43:02Z" message: 'Created pod: success-200-isvc-378f2-predictor-8bbd48d54-vp5rd' metadata: creationTimestamp: "2026-05-26T11:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:02Z" name: success-200-isvc-378f2-predictor-8bbd48d54.18b31ad5a5621630 namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: 8344baa6-96c2-48b8-b5ff-2a1b65862784 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:43:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-378f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: 16bda86e-1ff7-425f-8be1-8c5e11cbdb56 kind: Event lastTimestamp: "2026-05-26T11:43:02Z" message: Scaled up replica set success-200-isvc-378f2-predictor-8bbd48d54 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:02Z" name: success-200-isvc-378f2-predictor.18b31ad59f5a9fa1 namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: fd7e4187-2300-4787-babe-92e79731e9a3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:43:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-378f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: 18eef21c-f5f8-4d17-b552-f2b1e9b23a7d kind: Event lastTimestamp: "2026-05-26T11:43:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:32Z" name: success-200-isvc-378f2-predictor.18b31ad9253b2a10 namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: 3ea6b4cf-609c-4029-90a5-4c37685c4b6a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:43:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-378f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: 18eef21c-f5f8-4d17-b552-f2b1e9b23a7d kind: Event lastTimestamp: "2026-05-26T11:43:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:43:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:43:32Z" name: success-200-isvc-378f2-predictor.18b31ad925aa1b2e namespace: kserve-ci-e2e-test resourceVersion: "16351" uid: b53da347-8922-4288-8cda-94f482499aef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:43:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-378f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 18eef21c-f5f8-4d17-b552-f2b1e9b23a7d kind: Event lastTimestamp: "2026-05-26T11:44:32Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:43:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:32Z" name: success-200-isvc-378f2-predictor.18b31ae023d53ea2 namespace: kserve-ci-e2e-test resourceVersion: "16914" uid: ac321445-fa90-49e6-909b-6afabe4199b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-05-26T11:43:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-378f2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 18eef21c-f5f8-4d17-b552-f2b1e9b23a7d kind: Event lastTimestamp: "2026-05-26T11:44:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:43:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:32Z" name: success-200-isvc-378f2-predictor.18b31ae024401ee1 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: af0dd718-2da6-4159-bfa8-69dc4dcba88d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:43:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-378f2 namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: b43fc319-761c-4715-a6bf-3a4aebda7137 kind: Event lastTimestamp: "2026-05-26T11:44:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:43:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:04Z" name: success-200-isvc-378f2.18b31ad5a59a1d1b namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: b0b11bea-797a-4006-afe5-e24d5468cb19 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-378f2 namespace: kserve-ci-e2e-test resourceVersion: "16619" uid: b43fc319-761c-4715-a6bf-3a4aebda7137 kind: Event lastTimestamp: "2026-05-26T11:44:04Z" message: InferenceService [success-200-isvc-378f2] is Ready metadata: creationTimestamp: "2026-05-26T11:44:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:04Z" name: success-200-isvc-378f2.18b31ae406f518fc namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: e8e26e6c-abb2-4616-93ff-1841cb209b74 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:55:28.192863Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v namespace: kserve-ci-e2e-test resourceVersion: "22677" uid: 65e63c61-bd7f-4e6b-b2f8-779a68944c37 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v to ip-10-0-135-71.ec2.internal metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:55:28Z" name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v.18b31b833b68e063 namespace: kserve-ci-e2e-test resourceVersion: "22681" uid: a39553dc-74e1-4fa3-becb-28db1294ab61 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v namespace: kserve-ci-e2e-test resourceVersion: "22678" uid: 65e63c61-bd7f-4e6b-b2f8-779a68944c37 kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-3c458-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:28Z" name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v.18b31b8345351add namespace: kserve-ci-e2e-test resourceVersion: "22728" uid: 34fe0f01-f910-4609-b142-c684f91d305d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v namespace: kserve-ci-e2e-test resourceVersion: "22692" uid: 65e63c61-bd7f-4e6b-b2f8-779a68944c37 kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:55:29Z" name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v.18b31b837965f19f namespace: kserve-ci-e2e-test resourceVersion: "22751" uid: 7bd2f45c-96fe-415b-ab55-b514568eb2de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v namespace: kserve-ci-e2e-test resourceVersion: "22678" uid: 65e63c61-bd7f-4e6b-b2f8-779a68944c37 kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:29Z" name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v.18b31b837ac0ccd4 namespace: kserve-ci-e2e-test resourceVersion: "22754" uid: a1734acb-8af6-4388-b3a7-065271c82811 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v namespace: kserve-ci-e2e-test resourceVersion: "22678" uid: 65e63c61-bd7f-4e6b-b2f8-779a68944c37 kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:29Z" name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v.18b31b837e61659e namespace: kserve-ci-e2e-test resourceVersion: "22755" uid: 6b55cbe7-5932-4c0b-8f89-e44c96695c44 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v namespace: kserve-ci-e2e-test resourceVersion: "22678" uid: 65e63c61-bd7f-4e6b-b2f8-779a68944c37 kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:29Z" name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v.18b31b837ed768a9 namespace: kserve-ci-e2e-test resourceVersion: "22756" uid: 2aad9c8d-8fc2-486e-828e-51eb02d82e5b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v namespace: kserve-ci-e2e-test resourceVersion: "22678" uid: 65e63c61-bd7f-4e6b-b2f8-779a68944c37 kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:29Z" name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v.18b31b837ee330fd namespace: kserve-ci-e2e-test resourceVersion: "22757" uid: 35cbd106-a187-4a5c-a34f-dc7954249e2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v namespace: kserve-ci-e2e-test resourceVersion: "22678" uid: 65e63c61-bd7f-4e6b-b2f8-779a68944c37 kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:29Z" name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v.18b31b8381d4bdca namespace: kserve-ci-e2e-test resourceVersion: "22762" uid: 952ae4cd-8df7-481f-89ec-de9256d53b6f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v namespace: kserve-ci-e2e-test resourceVersion: "22678" uid: 65e63c61-bd7f-4e6b-b2f8-779a68944c37 kind: Event lastTimestamp: "2026-05-26T11:55:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:29Z" name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v.18b31b83824ea92a namespace: kserve-ci-e2e-test resourceVersion: "22763" uid: d3d78c5b-2efb-40cb-a952-684cff5c36c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:55:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v namespace: kserve-ci-e2e-test resourceVersion: "22678" uid: 65e63c61-bd7f-4e6b-b2f8-779a68944c37 kind: Event lastTimestamp: "2026-05-26T11:56:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:56:06Z" name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v.18b31b83da58a2a9 namespace: kserve-ci-e2e-test resourceVersion: "23140" uid: 40475e2f-17f1-4573-83c6-9f03b0831fc5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v namespace: kserve-ci-e2e-test resourceVersion: "22678" uid: 65e63c61-bd7f-4e6b-b2f8-779a68944c37 kind: Event lastTimestamp: "2026-05-26T12:04:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T12:04:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:42Z" name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v.18b31c046320858a namespace: kserve-ci-e2e-test resourceVersion: "27293" uid: 0bc1459c-cb1c-4f38-9c81-fc1346e01eef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v namespace: kserve-ci-e2e-test resourceVersion: "22678" uid: 65e63c61-bd7f-4e6b-b2f8-779a68944c37 kind: Event lastTimestamp: "2026-05-26T12:04:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:04:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:42Z" name: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v.18b31c046320f5ac namespace: kserve-ci-e2e-test resourceVersion: "27296" uid: d77b466b-e791-4980-aabf-0665073cb74e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3c458-predictor-7fc8474d4d namespace: kserve-ci-e2e-test resourceVersion: "22665" uid: 93aa4c62-caff-44f7-973b-989f02acf182 kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: 'Created pod: success-200-isvc-3c458-predictor-7fc8474d4d-jdr7v' metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:28Z" name: success-200-isvc-3c458-predictor-7fc8474d4d.18b31b833b0a7547 namespace: kserve-ci-e2e-test resourceVersion: "22679" uid: e42007c6-ae82-4cfc-9448-896cc9fe04c7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3c458-predictor namespace: kserve-ci-e2e-test resourceVersion: "22658" uid: 480d1310-46a9-4d6a-97e2-aabdaa9e8b1b kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: Scaled up replica set success-200-isvc-3c458-predictor-7fc8474d4d from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:28Z" name: success-200-isvc-3c458-predictor.18b31b833928dff7 namespace: kserve-ci-e2e-test resourceVersion: "22668" uid: 9ad6a226-dcb7-457b-934a-f720406ac03b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:55:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c458-predictor namespace: kserve-ci-e2e-test resourceVersion: "22663" uid: 4ad38394-322a-47b7-8905-301ae909720a kind: Event lastTimestamp: "2026-05-26T11:55:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:58Z" name: success-200-isvc-3c458-predictor.18b31b86b81982a7 namespace: kserve-ci-e2e-test resourceVersion: "23026" uid: af5175d4-4f1b-4062-90f0-dbf490d2ff49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:55:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c458-predictor namespace: kserve-ci-e2e-test resourceVersion: "22663" uid: 4ad38394-322a-47b7-8905-301ae909720a kind: Event lastTimestamp: "2026-05-26T11:55:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:58Z" name: success-200-isvc-3c458-predictor.18b31b86b8652f88 namespace: kserve-ci-e2e-test resourceVersion: "23027" uid: 87ac0eef-bc7f-4da4-ba59-36c307451632 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:56:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c458-predictor namespace: kserve-ci-e2e-test resourceVersion: "22889" uid: 4ad38394-322a-47b7-8905-301ae909720a kind: Event lastTimestamp: "2026-05-26T11:56:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:28Z" name: success-200-isvc-3c458-predictor.18b31b8db670c34c namespace: kserve-ci-e2e-test resourceVersion: "23327" uid: 510f8068-96c6-4ad8-a89a-6a6aaba43080 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:56:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c458-predictor namespace: kserve-ci-e2e-test resourceVersion: "22889" uid: 4ad38394-322a-47b7-8905-301ae909720a kind: Event lastTimestamp: "2026-05-26T11:56:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:56:28Z" name: success-200-isvc-3c458-predictor.18b31b8db6d6f3a1 namespace: kserve-ci-e2e-test resourceVersion: "23330" uid: cdaf3052-b4a7-4dd8-8393-0d01cf94535c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3c458 namespace: kserve-ci-e2e-test resourceVersion: "22645" uid: a1a12f27-5d92-43d9-829b-9cf7f0fbc891 kind: Event lastTimestamp: "2026-05-26T11:56:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:56:17Z" name: success-200-isvc-3c458.18b31b8339091b81 namespace: kserve-ci-e2e-test resourceVersion: "23237" uid: 2170bb62-ca47-4a11-a5ab-9ad4ad3061f2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3c458 namespace: kserve-ci-e2e-test resourceVersion: "22645" uid: a1a12f27-5d92-43d9-829b-9cf7f0fbc891 kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: 'Failed to update status for InferenceService "success-200-isvc-3c458": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3c458": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:28Z" name: success-200-isvc-3c458.18b31b83397757d1 namespace: kserve-ci-e2e-test resourceVersion: "22670" uid: 09290902-38ca-4f44-92aa-9a66df596ad6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3c458 namespace: kserve-ci-e2e-test resourceVersion: "22645" uid: a1a12f27-5d92-43d9-829b-9cf7f0fbc891 kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3c458": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:28Z" name: success-200-isvc-3c458.18b31b833977f0e6 namespace: kserve-ci-e2e-test resourceVersion: "22672" uid: 04d89850-97c9-4b89-9aa4-4f6b133002f1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:56:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3c458 namespace: kserve-ci-e2e-test resourceVersion: "23234" uid: a1a12f27-5d92-43d9-829b-9cf7f0fbc891 kind: Event lastTimestamp: "2026-05-26T11:56:16Z" message: InferenceService [success-200-isvc-3c458] is Ready metadata: creationTimestamp: "2026-05-26T11:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:56:16Z" name: success-200-isvc-3c458.18b31b8e96e3fc6b namespace: kserve-ci-e2e-test resourceVersion: "23235" uid: b60883c4-3603-452c-beea-d0efb9647df4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T12:05:23.664748Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-52624-predictor-68f4785b48-tj56s namespace: kserve-ci-e2e-test resourceVersion: "27812" uid: b4930a12-156e-40d8-9306-5b0598b02c9c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-52624-predictor-68f4785b48-tj56s to ip-10-0-135-71.ec2.internal metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T12:05:23Z" name: success-200-isvc-52624-predictor-68f4785b48-tj56s.18b31c0de04cbb34 namespace: kserve-ci-e2e-test resourceVersion: "27819" uid: 43ed63cf-6513-49bc-bcca-448820e797bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-52624-predictor-68f4785b48-tj56s namespace: kserve-ci-e2e-test resourceVersion: "27816" uid: b4930a12-156e-40d8-9306-5b0598b02c9c kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-52624-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:23Z" name: success-200-isvc-52624-predictor-68f4785b48-tj56s.18b31c0def8126fc namespace: kserve-ci-e2e-test resourceVersion: "27865" uid: 5a0e9115-e459-472d-88bf-58899c00aa94 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-52624-predictor-68f4785b48-tj56s namespace: kserve-ci-e2e-test resourceVersion: "27830" uid: b4930a12-156e-40d8-9306-5b0598b02c9c kind: Event lastTimestamp: "2026-05-26T12:05:24Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T12:05:24Z" name: success-200-isvc-52624-predictor-68f4785b48-tj56s.18b31c0e1e60e94a namespace: kserve-ci-e2e-test resourceVersion: "27880" uid: 4edaae6b-45f7-442e-925e-f42305771716 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52624-predictor-68f4785b48-tj56s namespace: kserve-ci-e2e-test resourceVersion: "27816" uid: b4930a12-156e-40d8-9306-5b0598b02c9c kind: Event lastTimestamp: "2026-05-26T12:05:24Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:24Z" name: success-200-isvc-52624-predictor-68f4785b48-tj56s.18b31c0e1fbb4c84 namespace: kserve-ci-e2e-test resourceVersion: "27882" uid: ab59d51d-4f79-4f68-ad04-1148d61d0455 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52624-predictor-68f4785b48-tj56s namespace: kserve-ci-e2e-test resourceVersion: "27816" uid: b4930a12-156e-40d8-9306-5b0598b02c9c kind: Event lastTimestamp: "2026-05-26T12:05:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:24Z" name: success-200-isvc-52624-predictor-68f4785b48-tj56s.18b31c0e233ac54c namespace: kserve-ci-e2e-test resourceVersion: "27884" uid: 9931032c-913a-43e0-a643-50294958d1ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52624-predictor-68f4785b48-tj56s namespace: kserve-ci-e2e-test resourceVersion: "27816" uid: b4930a12-156e-40d8-9306-5b0598b02c9c kind: Event lastTimestamp: "2026-05-26T12:05:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:24Z" name: success-200-isvc-52624-predictor-68f4785b48-tj56s.18b31c0e23ad4487 namespace: kserve-ci-e2e-test resourceVersion: "27885" uid: 34bcd523-4e5d-4967-8e06-ecf4a1ad9517 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-52624-predictor-68f4785b48-tj56s namespace: kserve-ci-e2e-test resourceVersion: "27816" uid: b4930a12-156e-40d8-9306-5b0598b02c9c kind: Event lastTimestamp: "2026-05-26T12:05:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:24Z" name: success-200-isvc-52624-predictor-68f4785b48-tj56s.18b31c0e23bb4d99 namespace: kserve-ci-e2e-test resourceVersion: "27886" uid: 8fcc7fc0-6831-4452-b76c-60ac58818a2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-52624-predictor-68f4785b48-tj56s namespace: kserve-ci-e2e-test resourceVersion: "27816" uid: b4930a12-156e-40d8-9306-5b0598b02c9c kind: Event lastTimestamp: "2026-05-26T12:05:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:24Z" name: success-200-isvc-52624-predictor-68f4785b48-tj56s.18b31c0e27434fa1 namespace: kserve-ci-e2e-test resourceVersion: "27890" uid: 062e285c-1f22-4503-b673-b6d5d40077d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-52624-predictor-68f4785b48-tj56s namespace: kserve-ci-e2e-test resourceVersion: "27816" uid: b4930a12-156e-40d8-9306-5b0598b02c9c kind: Event lastTimestamp: "2026-05-26T12:05:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:24Z" name: success-200-isvc-52624-predictor-68f4785b48-tj56s.18b31c0e27cb7597 namespace: kserve-ci-e2e-test resourceVersion: "27892" uid: 6b0813ec-1d93-4e0c-97a0-f669dd1e1528 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-05-26T12:05:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52624-predictor-68f4785b48-tj56s namespace: kserve-ci-e2e-test resourceVersion: "27816" uid: b4930a12-156e-40d8-9306-5b0598b02c9c kind: Event lastTimestamp: "2026-05-26T12:14:40Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T12:05:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:14:40Z" name: success-200-isvc-52624-predictor-68f4785b48-tj56s.18b31c0e2e3aca8e namespace: kserve-ci-e2e-test resourceVersion: "31937" uid: cd390a51-d31f-413e-bb50-b3b19b831541 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-52624-predictor-68f4785b48-tj56s namespace: kserve-ci-e2e-test resourceVersion: "27816" uid: b4930a12-156e-40d8-9306-5b0598b02c9c kind: Event lastTimestamp: "2026-05-26T12:14:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T12:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:14:38Z" name: success-200-isvc-52624-predictor-68f4785b48-tj56s.18b31c8f10e78605 namespace: kserve-ci-e2e-test resourceVersion: "31902" uid: b5c82f9d-cda3-40e7-8b69-7937ed9389cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:14:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-52624-predictor-68f4785b48-tj56s namespace: kserve-ci-e2e-test resourceVersion: "27816" uid: b4930a12-156e-40d8-9306-5b0598b02c9c kind: Event lastTimestamp: "2026-05-26T12:14:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:14:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:14:38Z" name: success-200-isvc-52624-predictor-68f4785b48-tj56s.18b31c8f10e82e31 namespace: kserve-ci-e2e-test resourceVersion: "31904" uid: d0937b4f-5ccb-4b8f-b1ad-c848d55389e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:14:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-52624-predictor-68f4785b48-tj56s namespace: kserve-ci-e2e-test resourceVersion: "27816" uid: b4930a12-156e-40d8-9306-5b0598b02c9c kind: Event lastTimestamp: "2026-05-26T12:14:40Z" message: 'Readiness probe failed: Get "https://10.133.0.44:8643/healthz": dial tcp 10.133.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T12:14:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:14:40Z" name: success-200-isvc-52624-predictor-68f4785b48-tj56s.18b31c8fa2985911 namespace: kserve-ci-e2e-test resourceVersion: "31936" uid: 93127639-0b1f-4e4e-b11b-8245d3d11a6c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-52624-predictor-68f4785b48 namespace: kserve-ci-e2e-test resourceVersion: "27802" uid: 49c465bf-a39e-44ed-9275-50fa9cc44d43 kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: 'Created pod: success-200-isvc-52624-predictor-68f4785b48-tj56s' metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:23Z" name: success-200-isvc-52624-predictor-68f4785b48.18b31c0ddfcf2825 namespace: kserve-ci-e2e-test resourceVersion: "27815" uid: 0359b607-0156-41b9-8d53-cbc05b181906 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-52624-predictor namespace: kserve-ci-e2e-test resourceVersion: "27797" uid: 944cc8c7-6c50-4c62-b260-1fe5dafba421 kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: Scaled up replica set success-200-isvc-52624-predictor-68f4785b48 from 0 to 1 metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:23Z" name: success-200-isvc-52624-predictor.18b31c0dde32965b namespace: kserve-ci-e2e-test resourceVersion: "27805" uid: 0bea8c56-9bf9-4452-b770-75df5da31143 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:05:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-52624-predictor namespace: kserve-ci-e2e-test resourceVersion: "27801" uid: 650e9cbc-6f9a-4e5f-9071-a5ff3e7fb0f6 kind: Event lastTimestamp: "2026-05-26T12:05:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:53Z" name: success-200-isvc-52624-predictor.18b31c115e4c7124 namespace: kserve-ci-e2e-test resourceVersion: "28211" uid: d2379261-6372-4c4e-8d9e-06ffce201af5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:05:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-52624-predictor namespace: kserve-ci-e2e-test resourceVersion: "27801" uid: 650e9cbc-6f9a-4e5f-9071-a5ff3e7fb0f6 kind: Event lastTimestamp: "2026-05-26T12:05:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:05:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:53Z" name: success-200-isvc-52624-predictor.18b31c115f1f1897 namespace: kserve-ci-e2e-test resourceVersion: "28212" uid: 261b10f9-e99d-41e8-b54d-e1807ef3bea7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:06:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-52624-predictor namespace: kserve-ci-e2e-test resourceVersion: "28047" uid: 650e9cbc-6f9a-4e5f-9071-a5ff3e7fb0f6 kind: Event lastTimestamp: "2026-05-26T12:06:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:06:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:06:23Z" name: success-200-isvc-52624-predictor.18b31c185ce60c33 namespace: kserve-ci-e2e-test resourceVersion: "28468" uid: fa826934-50c5-41ce-b196-e54c7777c160 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:06:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-52624-predictor namespace: kserve-ci-e2e-test resourceVersion: "28047" uid: 650e9cbc-6f9a-4e5f-9071-a5ff3e7fb0f6 kind: Event lastTimestamp: "2026-05-26T12:06:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:06:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:06:23Z" name: success-200-isvc-52624-predictor.18b31c185d568318 namespace: kserve-ci-e2e-test resourceVersion: "28473" uid: dd5882df-9ccd-45bc-9b84-0d3750ce7c8f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-52624 namespace: kserve-ci-e2e-test resourceVersion: "27780" uid: bb01b727-406f-4fce-9b85-85459d9292e9 kind: Event lastTimestamp: "2026-05-26T12:06:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:06:11Z" name: success-200-isvc-52624.18b31c0dde07ca06 namespace: kserve-ci-e2e-test resourceVersion: "28352" uid: d6ca32e0-acc6-44e6-aeea-b2eced87c29a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-52624 namespace: kserve-ci-e2e-test resourceVersion: "27780" uid: bb01b727-406f-4fce-9b85-85459d9292e9 kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: 'Failed to update status for InferenceService "success-200-isvc-52624": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-52624": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:23Z" name: success-200-isvc-52624.18b31c0dde7fb2bc namespace: kserve-ci-e2e-test resourceVersion: "27807" uid: 988b9443-7029-4619-896a-568e46f99839 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-52624 namespace: kserve-ci-e2e-test resourceVersion: "27780" uid: bb01b727-406f-4fce-9b85-85459d9292e9 kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-52624": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:23Z" name: success-200-isvc-52624.18b31c0dde801271 namespace: kserve-ci-e2e-test resourceVersion: "27809" uid: d2774774-65c5-4582-9af3-b347c9ed54fe reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:06:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-52624 namespace: kserve-ci-e2e-test resourceVersion: "28350" uid: bb01b727-406f-4fce-9b85-85459d9292e9 kind: Event lastTimestamp: "2026-05-26T12:06:11Z" message: InferenceService [success-200-isvc-52624] is Ready metadata: creationTimestamp: "2026-05-26T12:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:06:11Z" name: success-200-isvc-52624.18b31c18e99bc74b namespace: kserve-ci-e2e-test resourceVersion: "28351" uid: 606bdc50-bf79-4080-9e82-0cf8fc4493ef reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:53:48.055990Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw namespace: kserve-ci-e2e-test resourceVersion: "21341" uid: 56b04d4a-e2b3-43aa-a6f1-56035cdb156d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw to ip-10-0-135-71.ec2.internal metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:53:48Z" name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw.18b31b6beac9730f namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 270d643d-53a3-48d4-9069-cd14d34aa84a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: 56b04d4a-e2b3-43aa-a6f1-56035cdb156d kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:53:48Z" name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw.18b31b6c04971541 namespace: kserve-ci-e2e-test resourceVersion: "21392" uid: 083f9c86-d192-4b95-9940-7a1c34673884 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 56b04d4a-e2b3-43aa-a6f1-56035cdb156d kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw.18b31b6c066edf4b namespace: kserve-ci-e2e-test resourceVersion: "21395" uid: e1a9e846-0c35-47a3-af29-3fc0c8405318 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 56b04d4a-e2b3-43aa-a6f1-56035cdb156d kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw.18b31b6c09213e81 namespace: kserve-ci-e2e-test resourceVersion: "21396" uid: d13326c1-e145-4f7b-8382-3f0dde8f9b98 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 56b04d4a-e2b3-43aa-a6f1-56035cdb156d kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw.18b31b6c098cac2b namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 85da167c-4682-4431-ac91-32bbe30a5f05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 56b04d4a-e2b3-43aa-a6f1-56035cdb156d kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw.18b31b6c09991ef6 namespace: kserve-ci-e2e-test resourceVersion: "21399" uid: a790ce54-249c-4f3d-9410-a89414b9f676 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 56b04d4a-e2b3-43aa-a6f1-56035cdb156d kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw.18b31b6c0c232445 namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: b153869e-9e6d-4e12-b181-3030f8406653 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 56b04d4a-e2b3-43aa-a6f1-56035cdb156d kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:48Z" name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw.18b31b6c0c911c56 namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: 2081692f-050d-4cfb-868e-f8fc514122c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 56b04d4a-e2b3-43aa-a6f1-56035cdb156d kind: Event lastTimestamp: "2026-05-26T11:54:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:26Z" name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw.18b31b6c7c3cf84c namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: 52461580-61f9-4a4c-accb-681a88b91e89 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 56b04d4a-e2b3-43aa-a6f1-56035cdb156d kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:58Z" name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw.18b31b7c4992f3e1 namespace: kserve-ci-e2e-test resourceVersion: "22243" uid: e99561d7-e685-4ca0-a4d4-7a65dfd84cdc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 56b04d4a-e2b3-43aa-a6f1-56035cdb156d kind: Event lastTimestamp: "2026-05-26T11:54:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:54:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:58Z" name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw.18b31b7c4993fa8e namespace: kserve-ci-e2e-test resourceVersion: "22246" uid: 281aaf3a-dc87-48f2-a0b6-5b2979884cd6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 56b04d4a-e2b3-43aa-a6f1-56035cdb156d kind: Event lastTimestamp: "2026-05-26T11:55:01Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:01Z" name: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw.18b31b7d0478c1af namespace: kserve-ci-e2e-test resourceVersion: "22339" uid: 1654311e-609d-430e-8216-fa2601136907 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5380a-predictor-9b6ccbb7b namespace: kserve-ci-e2e-test resourceVersion: "21321" uid: d9411dc9-581b-4ed6-b90f-1d0162dc4810 kind: Event lastTimestamp: "2026-05-26T11:53:48Z" message: 'Created pod: success-200-isvc-5380a-predictor-9b6ccbb7b-xkmbw' metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:53:48Z" name: success-200-isvc-5380a-predictor-9b6ccbb7b.18b31b6bea699a85 namespace: kserve-ci-e2e-test resourceVersion: "21345" uid: 62d9208e-5bc3-4c9b-add2-851ffc28c0ea reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5380a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21318" uid: 89ce162d-a9b1-4e7c-8278-2d5e08ce340f kind: Event lastTimestamp: "2026-05-26T11:53:47Z" message: Scaled up replica set success-200-isvc-5380a-predictor-9b6ccbb7b from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:53:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:53:47Z" name: success-200-isvc-5380a-predictor.18b31b6be6bfc0a2 namespace: kserve-ci-e2e-test resourceVersion: "21322" uid: 06953f02-5d07-4ceb-9ee8-fddbef79b4ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:54:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5380a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: efc76153-be04-495a-913b-f3c40bc53919 kind: Event lastTimestamp: "2026-05-26T11:54:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:33Z" name: success-200-isvc-5380a-predictor.18b31b6f65eb5bfa namespace: kserve-ci-e2e-test resourceVersion: "21916" uid: 0fd5a925-3e15-43e6-95e7-96f3f823dbb6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:54:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5380a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: efc76153-be04-495a-913b-f3c40bc53919 kind: Event lastTimestamp: "2026-05-26T11:54:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:54:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:33Z" name: success-200-isvc-5380a-predictor.18b31b6f6655ee33 namespace: kserve-ci-e2e-test resourceVersion: "21917" uid: c1908f70-ac45-40aa-b826-598e14190dfe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5380a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: efc76153-be04-495a-913b-f3c40bc53919 kind: Event lastTimestamp: "2026-05-26T11:54:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:48Z" name: success-200-isvc-5380a-predictor.18b31b79e2e8cc4a namespace: kserve-ci-e2e-test resourceVersion: "22059" uid: 5f935af8-bbed-40d1-bc71-706345404787 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5380a-predictor namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: efc76153-be04-495a-913b-f3c40bc53919 kind: Event lastTimestamp: "2026-05-26T11:54:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:54:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:48Z" name: success-200-isvc-5380a-predictor.18b31b79e3582fac namespace: kserve-ci-e2e-test resourceVersion: "22061" uid: 5073b710-64f2-40c2-b0d1-d61b5759482d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:53:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5380a namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: b82597ef-2f92-4ec2-ae05-8ab892e85f70 kind: Event lastTimestamp: "2026-05-26T11:54:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:53:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:36Z" name: success-200-isvc-5380a.18b31b6be6c73ff2 namespace: kserve-ci-e2e-test resourceVersion: "21968" uid: 46e49993-7902-41ca-b2af-b2810f79051c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5380a namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: b82597ef-2f92-4ec2-ae05-8ab892e85f70 kind: Event lastTimestamp: "2026-05-26T11:53:47Z" message: 'Failed to update status for InferenceService "success-200-isvc-5380a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5380a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:53:47Z" name: success-200-isvc-5380a.18b31b6be75c9b83 namespace: kserve-ci-e2e-test resourceVersion: "21327" uid: 049617bf-06f5-4300-9f12-ce4027fc00cb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5380a namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: b82597ef-2f92-4ec2-ae05-8ab892e85f70 kind: Event lastTimestamp: "2026-05-26T11:53:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5380a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:53:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:53:48Z" name: success-200-isvc-5380a.18b31b6be75d0411 namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: f6bc7c40-f881-4394-b1da-75a444b8873d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5380a namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: b82597ef-2f92-4ec2-ae05-8ab892e85f70 kind: Event lastTimestamp: "2026-05-26T11:54:36Z" message: InferenceService [success-200-isvc-5380a] is Ready metadata: creationTimestamp: "2026-05-26T11:54:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:36Z" name: success-200-isvc-5380a.18b31b7737fb8855 namespace: kserve-ci-e2e-test resourceVersion: "21962" uid: 5b04772a-aa15-4f72-aaae-1974fce48bf7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:45:02.989928Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng namespace: kserve-ci-e2e-test resourceVersion: "17426" uid: 31cac73a-d9f6-4a14-b4fc-d090af94f5d7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5b0de-predictor-7945cf7494-zdzng to ip-10-0-135-71.ec2.internal metadata: creationTimestamp: "2026-05-26T11:45:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:45:02Z" name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng.18b31af1aa69345a namespace: kserve-ci-e2e-test resourceVersion: "17436" uid: 23a2647c-44f4-405d-8f2c-552a79a9373d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: 31cac73a-d9f6-4a14-b4fc-d090af94f5d7 kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-5b0de-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:45:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:03Z" name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng.18b31af1b744eeda namespace: kserve-ci-e2e-test resourceVersion: "17485" uid: 6c68dd58-7302-4fe5-9ba5-7e54ad2ef54d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng namespace: kserve-ci-e2e-test resourceVersion: "17448" uid: 31cac73a-d9f6-4a14-b4fc-d090af94f5d7 kind: Event lastTimestamp: "2026-05-26T11:45:04Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:45:04Z" name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng.18b31af1e80356e1 namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: c7ca6fec-29ae-4997-a05e-1baa7cc1f0a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: 31cac73a-d9f6-4a14-b4fc-d090af94f5d7 kind: Event lastTimestamp: "2026-05-26T11:45:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:04Z" name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng.18b31af1e94b6e11 namespace: kserve-ci-e2e-test resourceVersion: "17513" uid: b683b270-fcc6-407b-8f40-01a0e460fc50 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: 31cac73a-d9f6-4a14-b4fc-d090af94f5d7 kind: Event lastTimestamp: "2026-05-26T11:45:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:04Z" name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng.18b31af1ec72bfc6 namespace: kserve-ci-e2e-test resourceVersion: "17516" uid: 4072ad17-ff5b-42d4-9ab2-a6396c3ece94 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: 31cac73a-d9f6-4a14-b4fc-d090af94f5d7 kind: Event lastTimestamp: "2026-05-26T11:45:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:04Z" name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng.18b31af1ece3d33c namespace: kserve-ci-e2e-test resourceVersion: "17518" uid: 8e556bf9-08fa-4d3e-9938-280e7b9df033 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: 31cac73a-d9f6-4a14-b4fc-d090af94f5d7 kind: Event lastTimestamp: "2026-05-26T11:45:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:04Z" name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng.18b31af1ecefdbce namespace: kserve-ci-e2e-test resourceVersion: "17519" uid: 6d7d327b-334d-4cd0-8538-faca3ae2d763 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: 31cac73a-d9f6-4a14-b4fc-d090af94f5d7 kind: Event lastTimestamp: "2026-05-26T11:45:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:04Z" name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng.18b31af1f06fe5a1 namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: 19c87c21-4a7a-485d-bb1a-ed658b3ccc3c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: 31cac73a-d9f6-4a14-b4fc-d090af94f5d7 kind: Event lastTimestamp: "2026-05-26T11:45:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:45:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:04Z" name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng.18b31af1f0ddfb84 namespace: kserve-ci-e2e-test resourceVersion: "17521" uid: 94bff047-93b6-4039-9e1a-625dc55d2c45 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:45:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: 31cac73a-d9f6-4a14-b4fc-d090af94f5d7 kind: Event lastTimestamp: "2026-05-26T11:45:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:45:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:41Z" name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng.18b31af24ac2df73 namespace: kserve-ci-e2e-test resourceVersion: "17898" uid: 6556d1ae-0b95-4256-9f49-b07fed84ba21 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: 31cac73a-d9f6-4a14-b4fc-d090af94f5d7 kind: Event lastTimestamp: "2026-05-26T11:54:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:17Z" name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng.18b31b72ddb0ef83 namespace: kserve-ci-e2e-test resourceVersion: "21700" uid: 0d41ba14-aed0-4cae-8b0d-f46614ea9ba2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: 31cac73a-d9f6-4a14-b4fc-d090af94f5d7 kind: Event lastTimestamp: "2026-05-26T11:54:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:17Z" name: success-200-isvc-5b0de-predictor-7945cf7494-zdzng.18b31b72ddb228f4 namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: b433d8ea-15ed-440c-905a-9e0314336f70 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5b0de-predictor-7945cf7494 namespace: kserve-ci-e2e-test resourceVersion: "17410" uid: 1c8759bf-4b58-4a16-b6bf-76bbde6ecc26 kind: Event lastTimestamp: "2026-05-26T11:45:02Z" message: 'Created pod: success-200-isvc-5b0de-predictor-7945cf7494-zdzng' metadata: creationTimestamp: "2026-05-26T11:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:02Z" name: success-200-isvc-5b0de-predictor-7945cf7494.18b31af1a9fdfb77 namespace: kserve-ci-e2e-test resourceVersion: "17430" uid: 7b01f79a-8722-4ba8-a83e-3ab67d281fac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5b0de-predictor namespace: kserve-ci-e2e-test resourceVersion: "17403" uid: 7f051d68-5f38-4158-a3cc-e840d077aee2 kind: Event lastTimestamp: "2026-05-26T11:45:02Z" message: Scaled up replica set success-200-isvc-5b0de-predictor-7945cf7494 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:02Z" name: success-200-isvc-5b0de-predictor.18b31af1a7d97eb4 namespace: kserve-ci-e2e-test resourceVersion: "17414" uid: 936631dd-609d-4a05-97c9-127f24882f65 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:45:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5b0de-predictor namespace: kserve-ci-e2e-test resourceVersion: "17409" uid: 12d2118e-df1d-4d03-8afb-f8519488dfe2 kind: Event lastTimestamp: "2026-05-26T11:45:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:32Z" name: success-200-isvc-5b0de-predictor.18b31af526b7150e namespace: kserve-ci-e2e-test resourceVersion: "17782" uid: 2a023fa5-5140-4204-83bc-5b8fcf53f4b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:45:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5b0de-predictor namespace: kserve-ci-e2e-test resourceVersion: "17409" uid: 12d2118e-df1d-4d03-8afb-f8519488dfe2 kind: Event lastTimestamp: "2026-05-26T11:45:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:45:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:32Z" name: success-200-isvc-5b0de-predictor.18b31af527279818 namespace: kserve-ci-e2e-test resourceVersion: "17783" uid: 51a3fdd4-e17b-4c48-8086-d9e576f95882 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:45:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5b0de-predictor namespace: kserve-ci-e2e-test resourceVersion: "17643" uid: 12d2118e-df1d-4d03-8afb-f8519488dfe2 kind: Event lastTimestamp: "2026-05-26T11:46:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:45:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:18Z" name: success-200-isvc-5b0de-predictor.18b31afc251de853 namespace: kserve-ci-e2e-test resourceVersion: "18226" uid: 4a89374e-4d24-4095-acaa-a59b531c48e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:45:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5b0de-predictor namespace: kserve-ci-e2e-test resourceVersion: "17643" uid: 12d2118e-df1d-4d03-8afb-f8519488dfe2 kind: Event lastTimestamp: "2026-05-26T11:46:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:18Z" name: success-200-isvc-5b0de-predictor.18b31afc257ab28b namespace: kserve-ci-e2e-test resourceVersion: "18227" uid: 8ebf2d6a-2ee9-416a-8951-7c62601ba071 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:45:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5b0de namespace: kserve-ci-e2e-test resourceVersion: "17381" uid: bcf4b5c5-5bf5-4be8-9749-2c0e5b1eb8d4 kind: Event lastTimestamp: "2026-05-26T11:45:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:45:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:51Z" name: success-200-isvc-5b0de.18b31af1a7d2eb77 namespace: kserve-ci-e2e-test resourceVersion: "17993" uid: 49dab82f-2f2c-4351-af49-9ebcd71f222d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5b0de namespace: kserve-ci-e2e-test resourceVersion: "17991" uid: bcf4b5c5-5bf5-4be8-9749-2c0e5b1eb8d4 kind: Event lastTimestamp: "2026-05-26T11:45:51Z" message: InferenceService [success-200-isvc-5b0de] is Ready metadata: creationTimestamp: "2026-05-26T11:45:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:51Z" name: success-200-isvc-5b0de.18b31afd05957854 namespace: kserve-ci-e2e-test resourceVersion: "17992" uid: ad110bbd-0be3-48b0-bb57-aa665308b7e7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T12:04:13.418595Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-81833-predictor-5554ddb76-qccpc namespace: kserve-ci-e2e-test resourceVersion: "26920" uid: 25b5fa81-3e00-4738-b260-086600720195 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-81833-predictor-5554ddb76-qccpc to ip-10-0-135-71.ec2.internal metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T12:04:13Z" name: success-200-isvc-81833-predictor-5554ddb76-qccpc.18b31bfd854d85a4 namespace: kserve-ci-e2e-test resourceVersion: "26929" uid: 46b91bac-5ee0-45ed-8e2b-a1b50ae42711 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-81833-predictor-5554ddb76-qccpc namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 25b5fa81-3e00-4738-b260-086600720195 kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-81833-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:13Z" name: success-200-isvc-81833-predictor-5554ddb76-qccpc.18b31bfd9057a850 namespace: kserve-ci-e2e-test resourceVersion: "26958" uid: 9a151afc-fde0-47c7-896e-5e193cca3585 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-81833-predictor-5554ddb76-qccpc namespace: kserve-ci-e2e-test resourceVersion: "26942" uid: 25b5fa81-3e00-4738-b260-086600720195 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T12:04:14Z" name: success-200-isvc-81833-predictor-5554ddb76-qccpc.18b31bfdc301df12 namespace: kserve-ci-e2e-test resourceVersion: "26976" uid: cc2898a8-7ecd-4878-8067-1e2543906a18 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-81833-predictor-5554ddb76-qccpc namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 25b5fa81-3e00-4738-b260-086600720195 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:14Z" name: success-200-isvc-81833-predictor-5554ddb76-qccpc.18b31bfdc47a3673 namespace: kserve-ci-e2e-test resourceVersion: "26978" uid: 4c31cca4-1899-44e5-859a-a7bece4fb98b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-81833-predictor-5554ddb76-qccpc namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 25b5fa81-3e00-4738-b260-086600720195 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:14Z" name: success-200-isvc-81833-predictor-5554ddb76-qccpc.18b31bfdc7737492 namespace: kserve-ci-e2e-test resourceVersion: "26982" uid: bb0b3e63-65e2-4276-a14d-7ced28f93102 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-81833-predictor-5554ddb76-qccpc namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 25b5fa81-3e00-4738-b260-086600720195 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:14Z" name: success-200-isvc-81833-predictor-5554ddb76-qccpc.18b31bfdc80ab188 namespace: kserve-ci-e2e-test resourceVersion: "26984" uid: 670e9f2f-28c3-4c9e-a2ce-13e47546cf56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-81833-predictor-5554ddb76-qccpc namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 25b5fa81-3e00-4738-b260-086600720195 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:14Z" name: success-200-isvc-81833-predictor-5554ddb76-qccpc.18b31bfdc816f06d namespace: kserve-ci-e2e-test resourceVersion: "26985" uid: 2edf991c-44be-4fc0-ae3a-9939ab6691a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-81833-predictor-5554ddb76-qccpc namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 25b5fa81-3e00-4738-b260-086600720195 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:14Z" name: success-200-isvc-81833-predictor-5554ddb76-qccpc.18b31bfdcdf16f84 namespace: kserve-ci-e2e-test resourceVersion: "26990" uid: 8e1aff0d-94f2-467f-aa2e-463f5784b401 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-81833-predictor-5554ddb76-qccpc namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 25b5fa81-3e00-4738-b260-086600720195 kind: Event lastTimestamp: "2026-05-26T12:04:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:04:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:14Z" name: success-200-isvc-81833-predictor-5554ddb76-qccpc.18b31bfdce6a6199 namespace: kserve-ci-e2e-test resourceVersion: "26991" uid: b6577d92-19d0-4987-9bb4-8cc2729e27da reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T12:04:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-81833-predictor-5554ddb76-qccpc namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 25b5fa81-3e00-4738-b260-086600720195 kind: Event lastTimestamp: "2026-05-26T12:04:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T12:04:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:04:51Z" name: success-200-isvc-81833-predictor-5554ddb76-qccpc.18b31bfe0df7bbbd namespace: kserve-ci-e2e-test resourceVersion: "27443" uid: d11b9b95-7080-492f-aaee-7f91c32f4605 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-81833-predictor-5554ddb76-qccpc namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 25b5fa81-3e00-4738-b260-086600720195 kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:23Z" name: success-200-isvc-81833-predictor-5554ddb76-qccpc.18b31c0ddeaa77ef namespace: kserve-ci-e2e-test resourceVersion: "27808" uid: 81389af8-e660-40ff-b486-93f65319f7e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-81833-predictor-5554ddb76-qccpc namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 25b5fa81-3e00-4738-b260-086600720195 kind: Event lastTimestamp: "2026-05-26T12:05:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T12:05:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:23Z" name: success-200-isvc-81833-predictor-5554ddb76-qccpc.18b31c0ddeac1f14 namespace: kserve-ci-e2e-test resourceVersion: "27810" uid: 1b8c916d-7e7d-409d-8c38-b4e5057afbb5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-81833-predictor-5554ddb76-qccpc namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 25b5fa81-3e00-4738-b260-086600720195 kind: Event lastTimestamp: "2026-05-26T12:05:27Z" message: 'Readiness probe failed: Get "https://10.133.0.42:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-05-26T12:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:27Z" name: success-200-isvc-81833-predictor-5554ddb76-qccpc.18b31c0ed1d3841a namespace: kserve-ci-e2e-test resourceVersion: "27935" uid: 54145a4d-90c5-4070-a008-7297238ce49c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-81833-predictor-5554ddb76 namespace: kserve-ci-e2e-test resourceVersion: "26898" uid: 53c765b0-81d9-4b4c-8400-2e7a1cb97db0 kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: 'Created pod: success-200-isvc-81833-predictor-5554ddb76-qccpc' metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:04:13Z" name: success-200-isvc-81833-predictor-5554ddb76.18b31bfd84c50ede namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 985528a7-bbf5-4b4a-9081-8b7066bcdd68 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-81833-predictor namespace: kserve-ci-e2e-test resourceVersion: "26894" uid: 278df4ef-8e0e-4608-8c0f-c6ae05bbd32e kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: Scaled up replica set success-200-isvc-81833-predictor-5554ddb76 from 0 to 1 metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:04:13Z" name: success-200-isvc-81833-predictor.18b31bfd7fef6cf4 namespace: kserve-ci-e2e-test resourceVersion: "26900" uid: 2d3e6fb9-f444-4044-9d7e-95cf398fce2a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:04:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-81833-predictor namespace: kserve-ci-e2e-test resourceVersion: "26897" uid: 3e212bb5-3a3d-4aaf-955e-2b58e14e9f55 kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:04:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:04:43Z" name: success-200-isvc-81833-predictor.18b31c00ff03300c namespace: kserve-ci-e2e-test resourceVersion: "27342" uid: cd8d0ecc-c746-4e47-8fd7-6c13d420fdaf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:04:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-81833-predictor namespace: kserve-ci-e2e-test resourceVersion: "26897" uid: 3e212bb5-3a3d-4aaf-955e-2b58e14e9f55 kind: Event lastTimestamp: "2026-05-26T12:04:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:04:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:04:43Z" name: success-200-isvc-81833-predictor.18b31c00ff70b1d1 namespace: kserve-ci-e2e-test resourceVersion: "27343" uid: b5af35d0-9deb-4f4b-9dc5-f3ae72d506ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:04:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-81833-predictor namespace: kserve-ci-e2e-test resourceVersion: "27109" uid: 3e212bb5-3a3d-4aaf-955e-2b58e14e9f55 kind: Event lastTimestamp: "2026-05-26T12:05:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:13Z" name: success-200-isvc-81833-predictor.18b31c07fd1ef6d0 namespace: kserve-ci-e2e-test resourceVersion: "27642" uid: b19deb18-09be-4921-8816-9ec623d68f82 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T12:04:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-81833-predictor namespace: kserve-ci-e2e-test resourceVersion: "27109" uid: 3e212bb5-3a3d-4aaf-955e-2b58e14e9f55 kind: Event lastTimestamp: "2026-05-26T12:05:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T12:04:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:13Z" name: success-200-isvc-81833-predictor.18b31c07fd8c9ff5 namespace: kserve-ci-e2e-test resourceVersion: "27644" uid: 29425d37-3059-41e4-8c2c-2dc530daa441 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-81833 namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: 758cabd0-cc13-4ef2-9216-7bee09c3808c kind: Event lastTimestamp: "2026-05-26T12:05:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:01Z" name: success-200-isvc-81833.18b31bfd7ff98aaf namespace: kserve-ci-e2e-test resourceVersion: "27551" uid: 3315a10d-7ee1-46f3-80bd-b14ec2752204 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-81833 namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: 758cabd0-cc13-4ef2-9216-7bee09c3808c kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: 'Failed to update status for InferenceService "success-200-isvc-81833": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-81833": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:04:13Z" name: success-200-isvc-81833.18b31bfd80e5b223 namespace: kserve-ci-e2e-test resourceVersion: "26902" uid: 3514d400-ad92-4381-a5b8-63de3cfa8565 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:04:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-81833 namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: 758cabd0-cc13-4ef2-9216-7bee09c3808c kind: Event lastTimestamp: "2026-05-26T12:04:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-81833": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:04:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:04:13Z" name: success-200-isvc-81833.18b31bfd80e60b10 namespace: kserve-ci-e2e-test resourceVersion: "26904" uid: 491bee6c-f324-43db-bf66-11b522e65683 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-81833 namespace: kserve-ci-e2e-test resourceVersion: "27548" uid: 758cabd0-cc13-4ef2-9216-7bee09c3808c kind: Event lastTimestamp: "2026-05-26T12:05:01Z" message: InferenceService [success-200-isvc-81833] is Ready metadata: creationTimestamp: "2026-05-26T12:05:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:01Z" name: success-200-isvc-81833.18b31c08c9e9abb1 namespace: kserve-ci-e2e-test resourceVersion: "27549" uid: c0a11887-22e0-40ca-892d-436cc9554c08 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:44:33.080250Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx namespace: kserve-ci-e2e-test resourceVersion: "16983" uid: 538ee6b7-39e3-44a1-a6aa-ce07fca82bc5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-90450-predictor-6d6789cd9d-flxmx to ip-10-0-135-71.ec2.internal metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:44:33Z" name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx.18b31aeab3a7a886 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: f2a8e885-8eed-4ac7-ad69-54a2bf6ac810 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 538ee6b7-39e3-44a1-a6aa-ce07fca82bc5 kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-90450-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:33Z" name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx.18b31aeabda88182 namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 47476f24-15cf-4632-9f76-e427f9f958fa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx namespace: kserve-ci-e2e-test resourceVersion: "16999" uid: 538ee6b7-39e3-44a1-a6aa-ce07fca82bc5 kind: Event lastTimestamp: "2026-05-26T11:44:34Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:44:34Z" name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx.18b31aeaf0e8edf0 namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 55bc9f53-5574-4c74-ae94-b6c9f2a24ab7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 538ee6b7-39e3-44a1-a6aa-ce07fca82bc5 kind: Event lastTimestamp: "2026-05-26T11:44:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:34Z" name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx.18b31aeaf3a84dff namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 6e32b867-f9cc-454a-bddc-336d831e86f0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 538ee6b7-39e3-44a1-a6aa-ce07fca82bc5 kind: Event lastTimestamp: "2026-05-26T11:44:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:34Z" name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx.18b31aeaf6c29c8c namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: a27a5bc1-b59e-4ba0-8411-2e92164f689c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 538ee6b7-39e3-44a1-a6aa-ce07fca82bc5 kind: Event lastTimestamp: "2026-05-26T11:44:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:34Z" name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx.18b31aeaf7372e14 namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: 0a79fd35-a643-41b9-bc1f-0f3b5b882aa2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 538ee6b7-39e3-44a1-a6aa-ce07fca82bc5 kind: Event lastTimestamp: "2026-05-26T11:44:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:34Z" name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx.18b31aeaf746e37e namespace: kserve-ci-e2e-test resourceVersion: "17064" uid: b1f000c9-15bd-4102-acea-b721ae09eb83 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 538ee6b7-39e3-44a1-a6aa-ce07fca82bc5 kind: Event lastTimestamp: "2026-05-26T11:44:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:34Z" name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx.18b31aeafaa8b889 namespace: kserve-ci-e2e-test resourceVersion: "17065" uid: 088f323d-5c3c-4445-82d6-ad4d34e3200b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 538ee6b7-39e3-44a1-a6aa-ce07fca82bc5 kind: Event lastTimestamp: "2026-05-26T11:44:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:44:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:34Z" name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx.18b31aeafb175600 namespace: kserve-ci-e2e-test resourceVersion: "17066" uid: 1925a8ec-6cab-4a8a-8ed0-0e18c903470c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:44:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 538ee6b7-39e3-44a1-a6aa-ce07fca82bc5 kind: Event lastTimestamp: "2026-05-26T11:45:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:44:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:11Z" name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx.18b31aeb481abfd8 namespace: kserve-ci-e2e-test resourceVersion: "17593" uid: 87d319d0-b836-4e42-a05c-75cf2b069bb5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 538ee6b7-39e3-44a1-a6aa-ce07fca82bc5 kind: Event lastTimestamp: "2026-05-26T11:53:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:53:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:47Z" name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx.18b31b6be6cbdfb5 namespace: kserve-ci-e2e-test resourceVersion: "21324" uid: 04cd2310-078c-4b85-8147-6e67d918a696 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx namespace: kserve-ci-e2e-test resourceVersion: "16988" uid: 538ee6b7-39e3-44a1-a6aa-ce07fca82bc5 kind: Event lastTimestamp: "2026-05-26T11:53:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:53:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:47Z" name: success-200-isvc-90450-predictor-6d6789cd9d-flxmx.18b31b6be6cca223 namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: b331530d-ad2f-4828-9c9f-607e3dd20019 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-90450-predictor-6d6789cd9d namespace: kserve-ci-e2e-test resourceVersion: "16965" uid: 1da9fb13-e385-4eaa-aba4-c8f52850e8ed kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: 'Created pod: success-200-isvc-90450-predictor-6d6789cd9d-flxmx' metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:33Z" name: success-200-isvc-90450-predictor-6d6789cd9d.18b31aeab3149560 namespace: kserve-ci-e2e-test resourceVersion: "16987" uid: 1fbc530f-ac9a-45c7-8c2a-272468249195 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-90450-predictor namespace: kserve-ci-e2e-test resourceVersion: "16962" uid: 6d16af0c-c062-494a-afc9-d1f6d4329ea5 kind: Event lastTimestamp: "2026-05-26T11:44:33Z" message: Scaled up replica set success-200-isvc-90450-predictor-6d6789cd9d from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:33Z" name: success-200-isvc-90450-predictor.18b31aeab10e9b23 namespace: kserve-ci-e2e-test resourceVersion: "16968" uid: ab02a3df-59eb-41e7-9b9b-44bd011c31af reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:44:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-90450-predictor namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: ba3e8d58-11a4-4746-9aa2-d13772d3616e kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:03Z" name: success-200-isvc-90450-predictor.18b31aee2f98a68b namespace: kserve-ci-e2e-test resourceVersion: "17465" uid: efbfeb7e-2c12-446a-9b85-c23d749044fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:44:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-90450-predictor namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: ba3e8d58-11a4-4746-9aa2-d13772d3616e kind: Event lastTimestamp: "2026-05-26T11:45:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:44:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:03Z" name: success-200-isvc-90450-predictor.18b31aee30157fd7 namespace: kserve-ci-e2e-test resourceVersion: "17467" uid: e85701b2-ab2c-4844-8e1c-c61cac6c2aec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:45:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-90450-predictor namespace: kserve-ci-e2e-test resourceVersion: "17190" uid: ba3e8d58-11a4-4746-9aa2-d13772d3616e kind: Event lastTimestamp: "2026-05-26T11:45:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:48Z" name: success-200-isvc-90450-predictor.18b31af52dad653e namespace: kserve-ci-e2e-test resourceVersion: "17945" uid: 7130ef25-33ca-4f6b-bf65-ecf3100b31df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T11:45:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-90450-predictor namespace: kserve-ci-e2e-test resourceVersion: "17190" uid: ba3e8d58-11a4-4746-9aa2-d13772d3616e kind: Event lastTimestamp: "2026-05-26T11:45:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:45:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:48Z" name: success-200-isvc-90450-predictor.18b31af52e1c95ae namespace: kserve-ci-e2e-test resourceVersion: "17946" uid: 9dd48dfc-d9a7-4310-a6d3-8092f9c7a21c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:44:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-90450 namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: f8c03a9c-c0e3-4b72-b29c-34bf64a6aa6e kind: Event lastTimestamp: "2026-05-26T11:45:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:44:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:21Z" name: success-200-isvc-90450.18b31aeab0d5c873 namespace: kserve-ci-e2e-test resourceVersion: "17695" uid: 199342f6-9635-4dc4-b069-3baf9c5d1711 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-90450 namespace: kserve-ci-e2e-test resourceVersion: "17692" uid: f8c03a9c-c0e3-4b72-b29c-34bf64a6aa6e kind: Event lastTimestamp: "2026-05-26T11:45:21Z" message: InferenceService [success-200-isvc-90450] is Ready metadata: creationTimestamp: "2026-05-26T11:45:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:21Z" name: success-200-isvc-90450.18b31af60386d071 namespace: kserve-ci-e2e-test resourceVersion: "17693" uid: 7b762f3c-4d01-4f3a-85f9-2a18999fcfd4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:54:17.933725Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x namespace: kserve-ci-e2e-test resourceVersion: "21702" uid: 6399324a-aa3d-4bd2-ae41-36df7c5ccb5b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x to ip-10-0-135-71.ec2.internal metadata: creationTimestamp: "2026-05-26T11:54:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:54:17Z" name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x.18b31b72dfa371f1 namespace: kserve-ci-e2e-test resourceVersion: "21712" uid: fc108ea4-c074-44c4-872c-73a0ad2c07cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 6399324a-aa3d-4bd2-ae41-36df7c5ccb5b kind: Event lastTimestamp: "2026-05-26T11:54:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-fbdd2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:18Z" name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x.18b31b72e6eda4bb namespace: kserve-ci-e2e-test resourceVersion: "21758" uid: e26b6f35-2238-4b20-a448-2f463023ddf5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: 6399324a-aa3d-4bd2-ae41-36df7c5ccb5b kind: Event lastTimestamp: "2026-05-26T11:54:18Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:54:18Z" name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x.18b31b731d78adee namespace: kserve-ci-e2e-test resourceVersion: "21786" uid: 20ab8c5a-c49c-4646-85db-92d5fd1a889f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 6399324a-aa3d-4bd2-ae41-36df7c5ccb5b kind: Event lastTimestamp: "2026-05-26T11:54:18Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1532" already present on machine metadata: creationTimestamp: "2026-05-26T11:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:18Z" name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x.18b31b731ede1f92 namespace: kserve-ci-e2e-test resourceVersion: "21788" uid: f2d5c91e-aa41-4b19-b4d2-336a31dd267d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 6399324a-aa3d-4bd2-ae41-36df7c5ccb5b kind: Event lastTimestamp: "2026-05-26T11:54:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-05-26T11:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:19Z" name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x.18b31b7321f53c6c namespace: kserve-ci-e2e-test resourceVersion: "21789" uid: 6c985095-78db-4700-9665-3af8779e93ce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 6399324a-aa3d-4bd2-ae41-36df7c5ccb5b kind: Event lastTimestamp: "2026-05-26T11:54:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-05-26T11:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:19Z" name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x.18b31b732270a4b2 namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 0919a984-1929-4018-9d7e-c08fc3bef1b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 6399324a-aa3d-4bd2-ae41-36df7c5ccb5b kind: Event lastTimestamp: "2026-05-26T11:54:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-05-26T11:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:19Z" name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x.18b31b73227d6d98 namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: 35174ee2-2f9b-4e2a-a2e5-965035b626ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 6399324a-aa3d-4bd2-ae41-36df7c5ccb5b kind: Event lastTimestamp: "2026-05-26T11:54:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-05-26T11:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:19Z" name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x.18b31b7325b376ee namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 34248e2a-5557-4999-9d82-b30944f38e35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 6399324a-aa3d-4bd2-ae41-36df7c5ccb5b kind: Event lastTimestamp: "2026-05-26T11:54:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:19Z" name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x.18b31b7326215e88 namespace: kserve-ci-e2e-test resourceVersion: "21794" uid: 5db17991-7296-40a8-86c3-1c610cb549c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:54:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 6399324a-aa3d-4bd2-ae41-36df7c5ccb5b kind: Event lastTimestamp: "2026-05-26T11:54:56Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-05-26T11:54:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:56Z" name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x.18b31b737eff225e namespace: kserve-ci-e2e-test resourceVersion: "22164" uid: eb007d01-9eba-4be2-a4d6-c2ed84b650ed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 6399324a-aa3d-4bd2-ae41-36df7c5ccb5b kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:28Z" name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x.18b31b833a1d753f namespace: kserve-ci-e2e-test resourceVersion: "22673" uid: 57908859-9c28-43f5-8be5-24c2248d3b90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 6399324a-aa3d-4bd2-ae41-36df7c5ccb5b kind: Event lastTimestamp: "2026-05-26T11:55:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-05-26T11:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:55:28Z" name: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x.18b31b833a238d41 namespace: kserve-ci-e2e-test resourceVersion: "22676" uid: 012c5f95-5bdb-49cf-91e1-b111d8e29495 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-71.ec2.internal source: component: kubelet host: ip-10-0-135-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fbdd2-predictor-6f8b675bfd namespace: kserve-ci-e2e-test resourceVersion: "21692" uid: ae82ebd8-905e-4cd6-bd4d-3e1501a68099 kind: Event lastTimestamp: "2026-05-26T11:54:17Z" message: 'Created pod: success-200-isvc-fbdd2-predictor-6f8b675bfd-s6q7x' metadata: creationTimestamp: "2026-05-26T11:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:17Z" name: success-200-isvc-fbdd2-predictor-6f8b675bfd.18b31b72ded78015 namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: 7830497c-ce35-46e1-8a8e-28e258d32d7d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fbdd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21686" uid: 468adf00-20c2-4c56-89e7-732e52033881 kind: Event lastTimestamp: "2026-05-26T11:54:17Z" message: Scaled up replica set success-200-isvc-fbdd2-predictor-6f8b675bfd from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:17Z" name: success-200-isvc-fbdd2-predictor.18b31b72dcd1989b namespace: kserve-ci-e2e-test resourceVersion: "21694" uid: 3bd87950-a52a-4be9-8130-b85d87989041 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:54:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbdd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21691" uid: 6255ec1a-1fb4-45b4-96da-f63f62491e5c kind: Event lastTimestamp: "2026-05-26T11:54:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:47Z" name: success-200-isvc-fbdd2-predictor.18b31b765bf12b34 namespace: kserve-ci-e2e-test resourceVersion: "22050" uid: e2a427f7-25b9-478a-92fd-12ceaa691321 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:54:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbdd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21691" uid: 6255ec1a-1fb4-45b4-96da-f63f62491e5c kind: Event lastTimestamp: "2026-05-26T11:54:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:54:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:54:47Z" name: success-200-isvc-fbdd2-predictor.18b31b765c71afea namespace: kserve-ci-e2e-test resourceVersion: "22051" uid: 0b1198a3-2779-4aa5-a4f2-09c8bb2ed30e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:55:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbdd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: 6255ec1a-1fb4-45b4-96da-f63f62491e5c kind: Event lastTimestamp: "2026-05-26T11:55:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:17Z" name: success-200-isvc-fbdd2-predictor.18b31b7d5a531bfa namespace: kserve-ci-e2e-test resourceVersion: "22520" uid: 42953ceb-41e8-4019-b89a-7026b7091d50 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-05-26T11:55:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fbdd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: 6255ec1a-1fb4-45b4-96da-f63f62491e5c kind: Event lastTimestamp: "2026-05-26T11:55:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:55:17Z" name: success-200-isvc-fbdd2-predictor.18b31b7d5aa82a14 namespace: kserve-ci-e2e-test resourceVersion: "22522" uid: 8bdef395-4cf0-4fd5-852c-a80adc24a5d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbdd2 namespace: kserve-ci-e2e-test resourceVersion: "21669" uid: 1062614c-bd62-4318-97e3-7d9213abcde2 kind: Event lastTimestamp: "2026-05-26T11:55:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-05-26T11:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:06Z" name: success-200-isvc-fbdd2.18b31b72dca91563 namespace: kserve-ci-e2e-test resourceVersion: "22410" uid: bc811593-5bc2-491d-b8c5-21368ec322e1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbdd2 namespace: kserve-ci-e2e-test resourceVersion: "21669" uid: 1062614c-bd62-4318-97e3-7d9213abcde2 kind: Event lastTimestamp: "2026-05-26T11:54:17Z" message: 'Failed to update status for InferenceService "success-200-isvc-fbdd2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fbdd2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:17Z" name: success-200-isvc-fbdd2.18b31b72dd1877d7 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 6f09fb5e-5023-4220-8179-b41e1861eb0e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:54:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbdd2 namespace: kserve-ci-e2e-test resourceVersion: "21669" uid: 1062614c-bd62-4318-97e3-7d9213abcde2 kind: Event lastTimestamp: "2026-05-26T11:54:17Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fbdd2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:54:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:54:17Z" name: success-200-isvc-fbdd2.18b31b72dd18c272 namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: c84e0af8-6c75-40d9-a756-264d03d8780b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:55:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fbdd2 namespace: kserve-ci-e2e-test resourceVersion: "22407" uid: 1062614c-bd62-4318-97e3-7d9213abcde2 kind: Event lastTimestamp: "2026-05-26T11:55:06Z" message: InferenceService [success-200-isvc-fbdd2] is Ready metadata: creationTimestamp: "2026-05-26T11:55:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:55:06Z" name: success-200-isvc-fbdd2.18b31b7e39ddb80b namespace: kserve-ci-e2e-test resourceVersion: "22408" uid: 3bef35ca-3340-4f19-8abd-4f1ed9fd706c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T12:05:42.989756Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-152bc-5b8ffd9c8b-kvgnq namespace: kserve-ci-e2e-test resourceVersion: "28095" uid: 6745d2d9-a284-42e7-8e35-a3e2d249de82 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-152bc-5b8ffd9c8b-kvgnq to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T12:05:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T12:05:42Z" name: switch-graph-152bc-5b8ffd9c8b-kvgnq.18b31c1260290376 namespace: kserve-ci-e2e-test resourceVersion: "28100" uid: e972a973-364b-4dff-9f4d-72579adfe4c0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:43Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-152bc-5b8ffd9c8b-kvgnq namespace: kserve-ci-e2e-test resourceVersion: "28097" uid: 6745d2d9-a284-42e7-8e35-a3e2d249de82 kind: Event lastTimestamp: "2026-05-26T12:05:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-152bc-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T12:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:43Z" name: switch-graph-152bc-5b8ffd9c8b-kvgnq.18b31c126a5246c7 namespace: kserve-ci-e2e-test resourceVersion: "28116" uid: 996e9ecf-deb9-4ce9-863e-4ec177b484dc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:44Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-152bc-5b8ffd9c8b-kvgnq namespace: kserve-ci-e2e-test resourceVersion: "28106" uid: 6745d2d9-a284-42e7-8e35-a3e2d249de82 kind: Event lastTimestamp: "2026-05-26T12:05:44Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T12:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T12:05:44Z" name: switch-graph-152bc-5b8ffd9c8b-kvgnq.18b31c129dac141a namespace: kserve-ci-e2e-test resourceVersion: "28123" uid: f2204508-e272-4e40-8610-5fcd7c7e918d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-152bc} kind: Pod name: switch-graph-152bc-5b8ffd9c8b-kvgnq namespace: kserve-ci-e2e-test resourceVersion: "28097" uid: 6745d2d9-a284-42e7-8e35-a3e2d249de82 kind: Event lastTimestamp: "2026-05-26T12:05:44Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-26T12:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:44Z" name: switch-graph-152bc-5b8ffd9c8b-kvgnq.18b31c129eedbbe3 namespace: kserve-ci-e2e-test resourceVersion: "28125" uid: 14c2eb41-c656-427a-b702-44a0386aa1b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-152bc} kind: Pod name: switch-graph-152bc-5b8ffd9c8b-kvgnq namespace: kserve-ci-e2e-test resourceVersion: "28097" uid: 6745d2d9-a284-42e7-8e35-a3e2d249de82 kind: Event lastTimestamp: "2026-05-26T12:05:44Z" message: 'Created container: switch-graph-152bc' metadata: creationTimestamp: "2026-05-26T12:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:44Z" name: switch-graph-152bc-5b8ffd9c8b-kvgnq.18b31c12a1b3ba78 namespace: kserve-ci-e2e-test resourceVersion: "28127" uid: 8f428809-bb88-4a91-8651-842ec2d805be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-152bc} kind: Pod name: switch-graph-152bc-5b8ffd9c8b-kvgnq namespace: kserve-ci-e2e-test resourceVersion: "28097" uid: 6745d2d9-a284-42e7-8e35-a3e2d249de82 kind: Event lastTimestamp: "2026-05-26T12:05:44Z" message: Started container switch-graph-152bc metadata: creationTimestamp: "2026-05-26T12:05:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:05:44Z" name: switch-graph-152bc-5b8ffd9c8b-kvgnq.18b31c12a23f5efe namespace: kserve-ci-e2e-test resourceVersion: "28128" uid: a889863b-88e3-44f0-8b82-38b8d155cc86 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:22:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-152bc} kind: Pod name: switch-graph-152bc-5b8ffd9c8b-kvgnq namespace: kserve-ci-e2e-test resourceVersion: "28097" uid: 6745d2d9-a284-42e7-8e35-a3e2d249de82 kind: Event lastTimestamp: "2026-05-26T12:22:02Z" message: Stopping container switch-graph-152bc metadata: creationTimestamp: "2026-05-26T12:22:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:22:02Z" name: switch-graph-152bc-5b8ffd9c8b-kvgnq.18b31cf667b49802 namespace: kserve-ci-e2e-test resourceVersion: "34966" uid: 3c385169-8137-4e75-92ce-8cc5cc448413 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-05-26T12:22:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-152bc} kind: Pod name: switch-graph-152bc-5b8ffd9c8b-kvgnq namespace: kserve-ci-e2e-test resourceVersion: "28097" uid: 6745d2d9-a284-42e7-8e35-a3e2d249de82 kind: Event lastTimestamp: "2026-05-26T12:22:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T12:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T12:22:25Z" name: switch-graph-152bc-5b8ffd9c8b-kvgnq.18b31cf72a49f04c namespace: kserve-ci-e2e-test resourceVersion: "35200" uid: 48f722cf-dfa0-437a-9017-9e585bea1c13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-152bc-5b8ffd9c8b namespace: kserve-ci-e2e-test resourceVersion: "28092" uid: ba8557c4-056a-4ad5-8a11-0399765c99f5 kind: Event lastTimestamp: "2026-05-26T12:05:42Z" message: 'Created pod: switch-graph-152bc-5b8ffd9c8b-kvgnq' metadata: creationTimestamp: "2026-05-26T12:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:42Z" name: switch-graph-152bc-5b8ffd9c8b.18b31c125fb53258 namespace: kserve-ci-e2e-test resourceVersion: "28096" uid: 971c202b-b9c0-4d80-9364-daa8327df6d2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-152bc namespace: kserve-ci-e2e-test resourceVersion: "28090" uid: 2bcdb007-5e36-4efc-af51-192be6817302 kind: Event lastTimestamp: "2026-05-26T12:05:42Z" message: Scaled up replica set switch-graph-152bc-5b8ffd9c8b from 0 to 1 metadata: creationTimestamp: "2026-05-26T12:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:05:42Z" name: switch-graph-152bc.18b31c125f205a70 namespace: kserve-ci-e2e-test resourceVersion: "28093" uid: 62ab817c-a358-4f65-83e3-dc9c81e6c977 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:43Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-152bc namespace: kserve-ci-e2e-test resourceVersion: "28108" uid: 8722ec7b-1d52-43ad-8813-0a8efc9c8ff4 kind: Event lastTimestamp: "2026-05-26T12:05:43Z" message: InferenceGraph [switch-graph-152bc] is Ready metadata: creationTimestamp: "2026-05-26T12:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:43Z" name: switch-graph-152bc.18b31c126423eca7 namespace: kserve-ci-e2e-test resourceVersion: "28110" uid: 752c2299-5891-48ec-832e-ec5c6913d999 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:43Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-152bc namespace: kserve-ci-e2e-test resourceVersion: "28083" uid: 8722ec7b-1d52-43ad-8813-0a8efc9c8ff4 kind: Event lastTimestamp: "2026-05-26T12:05:43Z" message: 'Failed to update status for InferenceGraph "switch-graph-152bc": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-152bc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:43Z" name: switch-graph-152bc.18b31c126677e555 namespace: kserve-ci-e2e-test resourceVersion: "28113" uid: 3fe5e204-96ae-460a-a933-474e2c3309a6 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T12:05:43Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-152bc namespace: kserve-ci-e2e-test resourceVersion: "28083" uid: 8722ec7b-1d52-43ad-8813-0a8efc9c8ff4 kind: Event lastTimestamp: "2026-05-26T12:05:43Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-152bc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T12:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T12:05:43Z" name: switch-graph-152bc.18b31c1266784279 namespace: kserve-ci-e2e-test resourceVersion: "28114" uid: 18f80b2a-1d44-4b7a-ac42-ad9619d4c5ae reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T12:05:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-152bc namespace: kserve-ci-e2e-test resourceVersion: "28091" uid: 986884e6-0eaf-41eb-beb4-44e720b9813b kind: Event lastTimestamp: "2026-05-26T12:06:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:06:28Z" name: switch-graph-152bc.18b31c15de10e861 namespace: kserve-ci-e2e-test resourceVersion: "28524" uid: 28db18f8-084b-46ad-81d3-4b3c7befa2de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-05-26T12:05:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-152bc namespace: kserve-ci-e2e-test resourceVersion: "28091" uid: 986884e6-0eaf-41eb-beb4-44e720b9813b kind: Event lastTimestamp: "2026-05-26T12:06:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T12:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T12:06:28Z" name: switch-graph-152bc.18b31c15de768f53 namespace: kserve-ci-e2e-test resourceVersion: "28525" uid: deaa9620-bab4-4a0c-8b96-18794d64e2ba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:44:22.697437Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-378f2-5985b954cf-x9762 namespace: kserve-ci-e2e-test resourceVersion: "16791" uid: e88eb924-4af1-4f62-8ba7-1afaf826f46d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-378f2-5985b954cf-x9762 to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:44:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:44:22Z" name: switch-graph-378f2-5985b954cf-x9762.18b31ae848ca7b9a namespace: kserve-ci-e2e-test resourceVersion: "16796" uid: cbd75266-d58d-43e2-a482-fadab2dac82a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:22Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-378f2-5985b954cf-x9762 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: e88eb924-4af1-4f62-8ba7-1afaf826f46d kind: Event lastTimestamp: "2026-05-26T11:44:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-378f2-serving-cert" not found' metadata: creationTimestamp: "2026-05-26T11:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:22Z" name: switch-graph-378f2-5985b954cf-x9762.18b31ae851a52791 namespace: kserve-ci-e2e-test resourceVersion: "16806" uid: 59f2827b-d94c-47f0-afca-7ac30890497f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:23Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-378f2-5985b954cf-x9762 namespace: kserve-ci-e2e-test resourceVersion: "16799" uid: e88eb924-4af1-4f62-8ba7-1afaf826f46d kind: Event lastTimestamp: "2026-05-26T11:44:23Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:44:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:44:23Z" name: switch-graph-378f2-5985b954cf-x9762.18b31ae8863d9020 namespace: kserve-ci-e2e-test resourceVersion: "16816" uid: f9c651fe-e407-4590-ac8b-084ec95f94af reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-378f2} kind: Pod name: switch-graph-378f2-5985b954cf-x9762 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: e88eb924-4af1-4f62-8ba7-1afaf826f46d kind: Event lastTimestamp: "2026-05-26T11:44:23Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" metadata: creationTimestamp: "2026-05-26T11:44:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:23Z" name: switch-graph-378f2-5985b954cf-x9762.18b31ae8878d0e55 namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: 203e0d53-719a-4e08-a2f0-a5f0cd49bd2a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-378f2} kind: Pod name: switch-graph-378f2-5985b954cf-x9762 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: e88eb924-4af1-4f62-8ba7-1afaf826f46d kind: Event lastTimestamp: "2026-05-26T11:44:25Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" in 2.005s (2.005s including waiting). Image size: 217997404 bytes.' metadata: creationTimestamp: "2026-05-26T11:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:25Z" name: switch-graph-378f2-5985b954cf-x9762.18b31ae8ff140b21 namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: f746678f-80c3-4bae-8e14-a9807ea78196 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-378f2} kind: Pod name: switch-graph-378f2-5985b954cf-x9762 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: e88eb924-4af1-4f62-8ba7-1afaf826f46d kind: Event lastTimestamp: "2026-05-26T11:44:25Z" message: 'Created container: switch-graph-378f2' metadata: creationTimestamp: "2026-05-26T11:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:25Z" name: switch-graph-378f2-5985b954cf-x9762.18b31ae90244a8d6 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 2b03a1f0-97ae-4ef5-8b7f-51836c4db4fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-378f2} kind: Pod name: switch-graph-378f2-5985b954cf-x9762 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: e88eb924-4af1-4f62-8ba7-1afaf826f46d kind: Event lastTimestamp: "2026-05-26T11:44:25Z" message: Started container switch-graph-378f2 metadata: creationTimestamp: "2026-05-26T11:44:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:25Z" name: switch-graph-378f2-5985b954cf-x9762.18b31ae902c42877 namespace: kserve-ci-e2e-test resourceVersion: "16834" uid: 9e0c9dfd-3eda-46e4-a391-e7368ec96398 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-378f2} kind: Pod name: switch-graph-378f2-5985b954cf-x9762 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: e88eb924-4af1-4f62-8ba7-1afaf826f46d kind: Event lastTimestamp: "2026-05-26T11:44:32Z" message: Stopping container switch-graph-378f2 metadata: creationTimestamp: "2026-05-26T11:44:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:44:32Z" name: switch-graph-378f2-5985b954cf-x9762.18b31aeaa89e1f15 namespace: kserve-ci-e2e-test resourceVersion: "16935" uid: 18a8db83-6c4e-4786-ac34-040fa7aeca47 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:44:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-378f2} kind: Pod name: switch-graph-378f2-5985b954cf-x9762 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: e88eb924-4af1-4f62-8ba7-1afaf826f46d kind: Event lastTimestamp: "2026-05-26T11:45:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T11:44:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:01Z" name: switch-graph-378f2-5985b954cf-x9762.18b31aeb95d51c91 namespace: kserve-ci-e2e-test resourceVersion: "17340" uid: 163beb06-4b35-40a5-820c-3eb97332649b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-378f2-5985b954cf namespace: kserve-ci-e2e-test resourceVersion: "16788" uid: f7e9779c-a1be-4bb4-8260-ed96e67d2f34 kind: Event lastTimestamp: "2026-05-26T11:44:22Z" message: 'Created pod: switch-graph-378f2-5985b954cf-x9762' metadata: creationTimestamp: "2026-05-26T11:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:22Z" name: switch-graph-378f2-5985b954cf.18b31ae84862478f namespace: kserve-ci-e2e-test resourceVersion: "16793" uid: 6070c04a-4efe-4949-b50d-77cde5d9c224 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-378f2 namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: e6dbf8fb-1302-43dd-aa24-914991e383cb kind: Event lastTimestamp: "2026-05-26T11:44:22Z" message: Scaled up replica set switch-graph-378f2-5985b954cf from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:44:22Z" name: switch-graph-378f2.18b31ae8479b02d1 namespace: kserve-ci-e2e-test resourceVersion: "16789" uid: dbcae7ff-5613-409c-9b88-775007dbeef3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:44:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-378f2 namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 00093cb2-f6ff-480a-9196-fb10ab13104e kind: Event lastTimestamp: "2026-05-26T11:44:22Z" message: InferenceGraph [switch-graph-378f2] is Ready metadata: creationTimestamp: "2026-05-26T11:44:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:44:22Z" name: switch-graph-378f2.18b31ae84cd0f3c4 namespace: kserve-ci-e2e-test resourceVersion: "16804" uid: ab4f67d9-2eaa-4b71-b83d-bdac44957dc8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-05-26T11:45:33.141453Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-90450-79bdccd478-dd87q namespace: kserve-ci-e2e-test resourceVersion: "17803" uid: 1a31ec7d-e904-45d7-a4e6-ae2c0120a78e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-90450-79bdccd478-dd87q to ip-10-0-129-97.ec2.internal metadata: creationTimestamp: "2026-05-26T11:45:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-05-26T11:45:33Z" name: switch-graph-90450-79bdccd478-dd87q.18b31af8af94e0c2 namespace: kserve-ci-e2e-test resourceVersion: "17808" uid: ebd12e55-ab8c-49f6-85ce-2489991b8035 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-695fc477cf-jcsh9 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:33Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-90450-79bdccd478-dd87q namespace: kserve-ci-e2e-test resourceVersion: "17812" uid: 1a31ec7d-e904-45d7-a4e6-ae2c0120a78e kind: Event lastTimestamp: "2026-05-26T11:45:33Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-05-26T11:45:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-05-26T11:45:33Z" name: switch-graph-90450-79bdccd478-dd87q.18b31af8c94f819f namespace: kserve-ci-e2e-test resourceVersion: "17825" uid: 43202495-1ecf-463d-b5e0-22630019e566 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-90450} kind: Pod name: switch-graph-90450-79bdccd478-dd87q namespace: kserve-ci-e2e-test resourceVersion: "17805" uid: 1a31ec7d-e904-45d7-a4e6-ae2c0120a78e kind: Event lastTimestamp: "2026-05-26T11:45:33Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:367463b1d3262f4416182663b2344c09f7ccedc3764563fd3d0aee32ec772aae" already present on machine metadata: creationTimestamp: "2026-05-26T11:45:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:33Z" name: switch-graph-90450-79bdccd478-dd87q.18b31af8ca8bed41 namespace: kserve-ci-e2e-test resourceVersion: "17827" uid: aafe829f-6e9d-4382-9f7d-77788ec45798 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-90450} kind: Pod name: switch-graph-90450-79bdccd478-dd87q namespace: kserve-ci-e2e-test resourceVersion: "17805" uid: 1a31ec7d-e904-45d7-a4e6-ae2c0120a78e kind: Event lastTimestamp: "2026-05-26T11:45:33Z" message: 'Created container: switch-graph-90450' metadata: creationTimestamp: "2026-05-26T11:45:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:33Z" name: switch-graph-90450-79bdccd478-dd87q.18b31af8cd2c2964 namespace: kserve-ci-e2e-test resourceVersion: "17828" uid: 3521d0c6-687f-4512-8967-7911d0d5ddfa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-90450} kind: Pod name: switch-graph-90450-79bdccd478-dd87q namespace: kserve-ci-e2e-test resourceVersion: "17805" uid: 1a31ec7d-e904-45d7-a4e6-ae2c0120a78e kind: Event lastTimestamp: "2026-05-26T11:45:33Z" message: Started container switch-graph-90450 metadata: creationTimestamp: "2026-05-26T11:45:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:45:33Z" name: switch-graph-90450-79bdccd478-dd87q.18b31af8cda093b9 namespace: kserve-ci-e2e-test resourceVersion: "17829" uid: b202d303-0a14-498c-a6a1-b7bce33b44a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:53:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-90450} kind: Pod name: switch-graph-90450-79bdccd478-dd87q namespace: kserve-ci-e2e-test resourceVersion: "17805" uid: 1a31ec7d-e904-45d7-a4e6-ae2c0120a78e kind: Event lastTimestamp: "2026-05-26T11:53:47Z" message: Stopping container switch-graph-90450 metadata: creationTimestamp: "2026-05-26T11:53:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:53:47Z" name: switch-graph-90450-79bdccd478-dd87q.18b31b6be076ab0d namespace: kserve-ci-e2e-test resourceVersion: "21293" uid: e40ad55b-a295-493a-a3ff-cefbf5c88b29 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-05-26T11:53:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-90450} kind: Pod name: switch-graph-90450-79bdccd478-dd87q namespace: kserve-ci-e2e-test resourceVersion: "17805" uid: 1a31ec7d-e904-45d7-a4e6-ae2c0120a78e kind: Event lastTimestamp: "2026-05-26T11:54:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-05-26T11:53:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-05-26T11:54:15Z" name: switch-graph-90450-79bdccd478-dd87q.18b31b6c64ba6dfa namespace: kserve-ci-e2e-test resourceVersion: "21625" uid: f7d2f200-90df-4ea3-86ee-d82bd3e7fe54 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-97.ec2.internal source: component: kubelet host: ip-10-0-129-97.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-90450-79bdccd478 namespace: kserve-ci-e2e-test resourceVersion: "17799" uid: 9b9d28aa-3117-4ccb-af44-084388695ccc kind: Event lastTimestamp: "2026-05-26T11:45:33Z" message: 'Created pod: switch-graph-90450-79bdccd478-dd87q' metadata: creationTimestamp: "2026-05-26T11:45:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:33Z" name: switch-graph-90450-79bdccd478.18b31af8af29c3ef namespace: kserve-ci-e2e-test resourceVersion: "17804" uid: 83843a62-a7b4-40fb-b5e8-4581a406085e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-90450 namespace: kserve-ci-e2e-test resourceVersion: "17795" uid: 9f32ea26-ad93-465b-8851-2ba51cb0bf6b kind: Event lastTimestamp: "2026-05-26T11:45:33Z" message: Scaled up replica set switch-graph-90450-79bdccd478 from 0 to 1 metadata: creationTimestamp: "2026-05-26T11:45:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:33Z" name: switch-graph-90450.18b31af8ae5d8850 namespace: kserve-ci-e2e-test resourceVersion: "17801" uid: ce1995a0-e21c-445c-af51-d0667d0d6a90 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-90450 namespace: kserve-ci-e2e-test resourceVersion: "17814" uid: 3376f1b2-5936-4bee-bc84-dd2b49d2bc8a kind: Event lastTimestamp: "2026-05-26T11:45:33Z" message: InferenceGraph [switch-graph-90450] is Ready metadata: creationTimestamp: "2026-05-26T11:45:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:33Z" name: switch-graph-90450.18b31af8b35b559f namespace: kserve-ci-e2e-test resourceVersion: "17816" uid: 4875daf8-8d97-4dbe-8e9e-ff7298e1b15a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-90450 namespace: kserve-ci-e2e-test resourceVersion: "17785" uid: 3376f1b2-5936-4bee-bc84-dd2b49d2bc8a kind: Event lastTimestamp: "2026-05-26T11:45:33Z" message: 'Failed to update status for InferenceGraph "switch-graph-90450": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-90450": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:45:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:33Z" name: switch-graph-90450.18b31af8b76714ac namespace: kserve-ci-e2e-test resourceVersion: "17821" uid: cc501930-a9f7-4bb6-84eb-1d81892fdbab reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-90450 namespace: kserve-ci-e2e-test resourceVersion: "17785" uid: 3376f1b2-5936-4bee-bc84-dd2b49d2bc8a kind: Event lastTimestamp: "2026-05-26T11:45:33Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-90450": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-05-26T11:45:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-05-26T11:45:33Z" name: switch-graph-90450.18b31af8b76776e7 namespace: kserve-ci-e2e-test resourceVersion: "17822" uid: 103a1446-724c-4634-aea3-34ef8a566903 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-90450 namespace: kserve-ci-e2e-test resourceVersion: "17798" uid: 44f4e2a4-4248-40d8-bea2-99e3758217cb kind: Event lastTimestamp: "2026-05-26T11:45:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:48Z" name: switch-graph-90450.18b31afc2cd9cd71 namespace: kserve-ci-e2e-test resourceVersion: "17947" uid: de99c959-a8e6-446a-a68c-09eb16d2ab37 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:45:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-90450 namespace: kserve-ci-e2e-test resourceVersion: "17798" uid: 44f4e2a4-4248-40d8-bea2-99e3758217cb kind: Event lastTimestamp: "2026-05-26T11:45:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-05-26T11:45:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:45:48Z" name: switch-graph-90450.18b31afc2d3ccab0 namespace: kserve-ci-e2e-test resourceVersion: "17949" uid: e1807a6c-f57c-48fe-a3ce-2b77e9ec8672 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-90450 namespace: kserve-ci-e2e-test resourceVersion: "17948" uid: 44f4e2a4-4248-40d8-bea2-99e3758217cb kind: Event lastTimestamp: "2026-05-26T11:46:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:03Z" name: switch-graph-90450.18b31affabf07068 namespace: kserve-ci-e2e-test resourceVersion: "18109" uid: 5491a1dd-3629-48b0-9158-3fdced5a6b2e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-05-26T11:46:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-90450 namespace: kserve-ci-e2e-test resourceVersion: "17948" uid: 44f4e2a4-4248-40d8-bea2-99e3758217cb kind: Event lastTimestamp: "2026-05-26T11:46:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-05-26T11:46:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-05-26T11:46:03Z" name: switch-graph-90450.18b31affac580404 namespace: kserve-ci-e2e-test resourceVersion: "18111" uid: 5ef7ef92-1bd3-4dd5-b9eb-5c7f10473ec0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "35224"