--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:52:27.232585Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: 7679d6ba-0118-46be-b76d-b37b54205a3a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-7acc8-6c9bbc6b6f-mqxtm to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:52:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:52:27Z" name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm.18be8dbd6ecd87f0 namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: 2e54c9e8-4189-43d3-81f5-8d9c9fb787fd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:27Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm namespace: kserve-ci-e2e-test resourceVersion: "21490" uid: 7679d6ba-0118-46be-b76d-b37b54205a3a kind: Event lastTimestamp: "2026-07-02T18:52:27Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:52:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:52:27Z" name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm.18be8dbd88f8127f namespace: kserve-ci-e2e-test resourceVersion: "21502" uid: f7924065-62db-4ace-bbb9-237f37c37ba6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7acc8} kind: Pod name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: 7679d6ba-0118-46be-b76d-b37b54205a3a kind: Event lastTimestamp: "2026-07-02T18:52:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:75c3c56106a5b8ed3fd4cb951cdacfbe184ab3e990f6497776c0954efdfa10b2" already present on machine metadata: creationTimestamp: "2026-07-02T18:52:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:27Z" name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm.18be8dbd8a37aa6e namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: 3dcddbcb-af63-4e42-9171-ec624f6a24d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7acc8} kind: Pod name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: 7679d6ba-0118-46be-b76d-b37b54205a3a kind: Event lastTimestamp: "2026-07-02T18:52:27Z" message: 'Created container: ensemble-graph-7acc8' metadata: creationTimestamp: "2026-07-02T18:52:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:27Z" name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm.18be8dbd8e2e8eb4 namespace: kserve-ci-e2e-test resourceVersion: "21505" uid: 0f087f31-25a7-49f8-a386-c708b91ca008 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7acc8} kind: Pod name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: 7679d6ba-0118-46be-b76d-b37b54205a3a kind: Event lastTimestamp: "2026-07-02T18:52:27Z" message: Started container ensemble-graph-7acc8 metadata: creationTimestamp: "2026-07-02T18:52:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:27Z" name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm.18be8dbd8ea28657 namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: e9a63306-3682-4904-a690-de03345302c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7acc8} kind: Pod name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: 7679d6ba-0118-46be-b76d-b37b54205a3a kind: Event lastTimestamp: "2026-07-02T19:00:41Z" message: Stopping container ensemble-graph-7acc8 metadata: creationTimestamp: "2026-07-02T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:41Z" name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm.18be8e3092aa7bc8 namespace: kserve-ci-e2e-test resourceVersion: "24977" uid: a811bdb3-c431-45f1-a4a4-d0aa5177d997 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T19:00:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7acc8} kind: Pod name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: 7679d6ba-0118-46be-b76d-b37b54205a3a kind: Event lastTimestamp: "2026-07-02T19:01:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T19:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:09Z" name: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm.18be8e313508c858 namespace: kserve-ci-e2e-test resourceVersion: "25578" uid: 43f46694-9838-4e7e-80c5-cc2448ceedf0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-7acc8-6c9bbc6b6f namespace: kserve-ci-e2e-test resourceVersion: "21476" uid: bb925e2e-4456-4faf-acfb-d420bff87c4d kind: Event lastTimestamp: "2026-07-02T18:52:27Z" message: 'Created pod: ensemble-graph-7acc8-6c9bbc6b6f-mqxtm' metadata: creationTimestamp: "2026-07-02T18:52:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:27Z" name: ensemble-graph-7acc8-6c9bbc6b6f.18be8dbd6e48b961 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 41069249-0169-463e-93b3-fd189163e318 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: c2a4c568-d9f7-4f7f-a023-f26d8ad83ca1 kind: Event lastTimestamp: "2026-07-02T18:52:27Z" message: Scaled up replica set ensemble-graph-7acc8-6c9bbc6b6f from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:52:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:27Z" name: ensemble-graph-7acc8.18be8dbd6db2fda7 namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 8a97f182-3ebe-4aa1-9d0d-ffdcf89255b3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:27Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "21496" uid: e4d650de-4bf7-4ab1-a610-346168a80785 kind: Event lastTimestamp: "2026-07-02T18:52:27Z" message: InferenceGraph [ensemble-graph-7acc8] is Ready metadata: creationTimestamp: "2026-07-02T18:52:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:52:27Z" name: ensemble-graph-7acc8.18be8dbd72ef701c namespace: kserve-ci-e2e-test resourceVersion: "21498" uid: aca7492c-4880-4f56-9d05-37befac1a486 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: a36f996f-e1ce-4c58-a597-829f36f93809 kind: Event lastTimestamp: "2026-07-02T18:52:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:52:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:42Z" name: ensemble-graph-7acc8.18be8dc0ec1ddb03 namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: 1dc4b5df-ef62-4bde-882b-b205bff30ba6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: a36f996f-e1ce-4c58-a597-829f36f93809 kind: Event lastTimestamp: "2026-07-02T18:52:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:52:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:42Z" name: ensemble-graph-7acc8.18be8dc0ec793328 namespace: kserve-ci-e2e-test resourceVersion: "21623" uid: 9ff651a6-4666-4b06-b7d1-b15b6fd443b1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "21621" uid: a36f996f-e1ce-4c58-a597-829f36f93809 kind: Event lastTimestamp: "2026-07-02T18:52:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:57Z" name: ensemble-graph-7acc8.18be8dc46b0a78fd namespace: kserve-ci-e2e-test resourceVersion: "21758" uid: e9c75760-db61-44a8-af44-d74f6224297b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "21621" uid: a36f996f-e1ce-4c58-a597-829f36f93809 kind: Event lastTimestamp: "2026-07-02T18:52:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:57Z" name: ensemble-graph-7acc8.18be8dc46b65680e namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: 8713fec5-baef-41a9-b7a6-978613788c3b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:51:16.930264Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e22cf-5c9b96dc46-s9n6z namespace: kserve-ci-e2e-test resourceVersion: "20513" uid: 196bedf0-44de-4568-8804-6506181b7e5f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-e22cf-5c9b96dc46-s9n6z to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:51:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:51:16Z" name: ensemble-graph-e22cf-5c9b96dc46-s9n6z.18be8dad10753b76 namespace: kserve-ci-e2e-test resourceVersion: "20518" uid: acfa41db-130c-48d9-9378-7af0eb5ec22c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:17Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e22cf-5c9b96dc46-s9n6z namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: 196bedf0-44de-4568-8804-6506181b7e5f kind: Event lastTimestamp: "2026-07-02T18:51:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-e22cf-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T18:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:17Z" name: ensemble-graph-e22cf-5c9b96dc46-s9n6z.18be8dad1c52d4e8 namespace: kserve-ci-e2e-test resourceVersion: "20531" uid: efbf28c7-5ebd-454b-8fac-8103f117881c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:17Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e22cf-5c9b96dc46-s9n6z namespace: kserve-ci-e2e-test resourceVersion: "20522" uid: 196bedf0-44de-4568-8804-6506181b7e5f kind: Event lastTimestamp: "2026-07-02T18:51:17Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:51:17Z" name: ensemble-graph-e22cf-5c9b96dc46-s9n6z.18be8dad4da7145b namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 4ef5d513-e4cb-48f4-9a3f-2da64a16748e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e22cf} kind: Pod name: ensemble-graph-e22cf-5c9b96dc46-s9n6z namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: 196bedf0-44de-4568-8804-6506181b7e5f kind: Event lastTimestamp: "2026-07-02T18:51:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:75c3c56106a5b8ed3fd4cb951cdacfbe184ab3e990f6497776c0954efdfa10b2" already present on machine metadata: creationTimestamp: "2026-07-02T18:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:17Z" name: ensemble-graph-e22cf-5c9b96dc46-s9n6z.18be8dad4ee78932 namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: 4a1aa2fb-6c0f-4a7b-89b0-737856df21a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e22cf} kind: Pod name: ensemble-graph-e22cf-5c9b96dc46-s9n6z namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: 196bedf0-44de-4568-8804-6506181b7e5f kind: Event lastTimestamp: "2026-07-02T18:51:18Z" message: 'Created container: ensemble-graph-e22cf' metadata: creationTimestamp: "2026-07-02T18:51:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:18Z" name: ensemble-graph-e22cf-5c9b96dc46-s9n6z.18be8dad51bf8af6 namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: a0ce740f-fa72-4450-9483-4de304c1344b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e22cf} kind: Pod name: ensemble-graph-e22cf-5c9b96dc46-s9n6z namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: 196bedf0-44de-4568-8804-6506181b7e5f kind: Event lastTimestamp: "2026-07-02T18:51:18Z" message: Started container ensemble-graph-e22cf metadata: creationTimestamp: "2026-07-02T18:51:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:18Z" name: ensemble-graph-e22cf-5c9b96dc46-s9n6z.18be8dad52310295 namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: 036c6640-b45d-472d-b5ca-065814d67f6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e22cf} kind: Pod name: ensemble-graph-e22cf-5c9b96dc46-s9n6z namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: 196bedf0-44de-4568-8804-6506181b7e5f kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: Stopping container ensemble-graph-e22cf metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:27Z" name: ensemble-graph-e22cf-5c9b96dc46-s9n6z.18be8daf6924c2d9 namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: 31a045bb-241d-4a07-a702-3dab3170b5f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:51:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e22cf} kind: Pod name: ensemble-graph-e22cf-5c9b96dc46-s9n6z namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: 196bedf0-44de-4568-8804-6506181b7e5f kind: Event lastTimestamp: "2026-07-02T18:51:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T18:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:54Z" name: ensemble-graph-e22cf-5c9b96dc46-s9n6z.18be8dafe6f681f2 namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: 2e1bf8e4-5a54-4c93-8efa-0a2b4d7397e6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-e22cf-5c9b96dc46 namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: bf5c8546-a70e-4c1d-9b6c-18709695700b kind: Event lastTimestamp: "2026-07-02T18:51:16Z" message: 'Created pod: ensemble-graph-e22cf-5c9b96dc46-s9n6z' metadata: creationTimestamp: "2026-07-02T18:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:16Z" name: ensemble-graph-e22cf-5c9b96dc46.18be8dad101089d0 namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 95e53aab-0ef8-4d6b-bc73-90700d65167d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-e22cf namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: 5a7c013d-0802-473f-b8b3-586e2fc69703 kind: Event lastTimestamp: "2026-07-02T18:51:16Z" message: Scaled up replica set ensemble-graph-e22cf-5c9b96dc46 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:16Z" name: ensemble-graph-e22cf.18be8dad0f552078 namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: a4209bee-eddb-424b-ba67-19ce51562973 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-e22cf namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: a6848cf2-1df1-4cce-9eca-44a903d76c63 kind: Event lastTimestamp: "2026-07-02T18:51:16Z" message: InferenceGraph [ensemble-graph-e22cf] is Ready metadata: creationTimestamp: "2026-07-02T18:51:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:16Z" name: ensemble-graph-e22cf.18be8dad142c150c namespace: kserve-ci-e2e-test resourceVersion: "20528" uid: bd82b6a3-e8b8-48f8-8bdf-20890da776bd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T19:01:22.078368Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4 namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: f89b4832-6ab6-47ea-b907-00ea9a6d239b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4 to ip-10-0-140-68.ec2.internal metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T19:01:22Z" name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4.18be8e39f6187a8a namespace: kserve-ci-e2e-test resourceVersion: "25776" uid: 00e85402-16d2-490c-b0d0-e00b0cae147e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4 namespace: kserve-ci-e2e-test resourceVersion: "25780" uid: f89b4832-6ab6-47ea-b907-00ea9a6d239b kind: Event lastTimestamp: "2026-07-02T19:01:22Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T19:01:22Z" name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4.18be8e3a0fa0acaf namespace: kserve-ci-e2e-test resourceVersion: "25798" uid: 3e0f0ee4-47be-4790-81af-13b94291c865 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: f89b4832-6ab6-47ea-b907-00ea9a6d239b kind: Event lastTimestamp: "2026-07-02T19:01:22Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:22Z" name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4.18be8e3a10d916f1 namespace: kserve-ci-e2e-test resourceVersion: "25800" uid: f1b341f8-a557-4bcc-8f23-9a74a54dc04a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: f89b4832-6ab6-47ea-b907-00ea9a6d239b kind: Event lastTimestamp: "2026-07-02T19:01:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:22Z" name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4.18be8e3a13f3f042 namespace: kserve-ci-e2e-test resourceVersion: "25801" uid: ff1f8e52-b2c1-4061-9cd4-eebf535fe916 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: f89b4832-6ab6-47ea-b907-00ea9a6d239b kind: Event lastTimestamp: "2026-07-02T19:01:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:22Z" name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4.18be8e3a14638d3c namespace: kserve-ci-e2e-test resourceVersion: "25803" uid: 53272dc1-fc67-4e1a-8a48-46a3b8401a9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: f89b4832-6ab6-47ea-b907-00ea9a6d239b kind: Event lastTimestamp: "2026-07-02T19:01:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:22Z" name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4.18be8e3a146c4768 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 7729bd09-a3f7-49ba-8208-64b082829b41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: f89b4832-6ab6-47ea-b907-00ea9a6d239b kind: Event lastTimestamp: "2026-07-02T19:01:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:22Z" name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4.18be8e3a17431b94 namespace: kserve-ci-e2e-test resourceVersion: "25806" uid: 671386d4-5008-41e6-8259-a380e4b33522 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: f89b4832-6ab6-47ea-b907-00ea9a6d239b kind: Event lastTimestamp: "2026-07-02T19:01:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:22Z" name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4.18be8e3a17b74e5f namespace: kserve-ci-e2e-test resourceVersion: "25807" uid: 0743cecb-2826-41dc-aa75-7c08ce9df218 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T19:01:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: f89b4832-6ab6-47ea-b907-00ea9a6d239b kind: Event lastTimestamp: "2026-07-02T19:02:00Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T19:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:02:00Z" name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4.18be8e3a816d2926 namespace: kserve-ci-e2e-test resourceVersion: "26364" uid: 27dc20bd-e3e8-4935-beb1-840051a6a7f5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:18:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: f89b4832-6ab6-47ea-b907-00ea9a6d239b kind: Event lastTimestamp: "2026-07-02T19:18:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T19:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:18:41Z" name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4.18be8f2c00621fb2 namespace: kserve-ci-e2e-test resourceVersion: "33491" uid: a0ff03ef-cd19-45ee-a926-6a574af82048 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:18:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: f89b4832-6ab6-47ea-b907-00ea9a6d239b kind: Event lastTimestamp: "2026-07-02T19:18:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:18:41Z" name: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4.18be8f2c0062a7c8 namespace: kserve-ci-e2e-test resourceVersion: "33492" uid: 62895f46-86ec-4d85-b846-9f30ce03565e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-208dc-predictor-7dd8d55cdb namespace: kserve-ci-e2e-test resourceVersion: "25765" uid: 59c2e499-9308-4fc5-aaf6-37a8cecb3457 kind: Event lastTimestamp: "2026-07-02T19:01:22Z" message: 'Created pod: error-404-isvc-208dc-predictor-7dd8d55cdb-fm9m4' metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:22Z" name: error-404-isvc-208dc-predictor-7dd8d55cdb.18be8e39f58dacb7 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: bfb30963-dfc8-4ab5-bd60-b417aeb7e7b9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-208dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25762" uid: 4a9a8c19-a85f-4f5e-bdb3-f32df6a4ea35 kind: Event lastTimestamp: "2026-07-02T19:01:22Z" message: Scaled up replica set error-404-isvc-208dc-predictor-7dd8d55cdb from 0 to 1 metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:22Z" name: error-404-isvc-208dc-predictor.18be8e39f417fed4 namespace: kserve-ci-e2e-test resourceVersion: "25766" uid: 03ba8476-a843-4872-b581-95179f7cf077 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T19:01:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-208dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25764" uid: 5f4cd20f-fcc8-48f4-abf5-77d66295fb8f kind: Event lastTimestamp: "2026-07-02T19:02:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:37Z" name: error-404-isvc-208dc-predictor.18be8e3d7290693f namespace: kserve-ci-e2e-test resourceVersion: "26713" uid: a49e1494-e0bb-4c56-9e64-fc7b75d605bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T19:01:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-208dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25764" uid: 5f4cd20f-fcc8-48f4-abf5-77d66295fb8f kind: Event lastTimestamp: "2026-07-02T19:02:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:37Z" name: error-404-isvc-208dc-predictor.18be8e3d72d05bbf namespace: kserve-ci-e2e-test resourceVersion: "26714" uid: 82925ead-2680-42cf-a1ac-0f6d6b5f38b7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T19:01:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-208dc namespace: kserve-ci-e2e-test resourceVersion: "25747" uid: eebd7921-cc2d-4a3e-9fc9-d037ae6a56d7 kind: Event lastTimestamp: "2026-07-02T19:02:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:02:10Z" name: error-404-isvc-208dc.18be8e39f42cbc4a namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: a5353a12-4ec5-48c6-9494-f59aa7965e28 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-208dc namespace: kserve-ci-e2e-test resourceVersion: "25747" uid: eebd7921-cc2d-4a3e-9fc9-d037ae6a56d7 kind: Event lastTimestamp: "2026-07-02T19:01:22Z" message: 'Failed to update status for InferenceService "error-404-isvc-208dc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-208dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:01:22Z" name: error-404-isvc-208dc.18be8e39f494c939 namespace: kserve-ci-e2e-test resourceVersion: "25769" uid: ee4db28f-27cc-45cb-8c51-f0b9625efa60 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-208dc namespace: kserve-ci-e2e-test resourceVersion: "25747" uid: eebd7921-cc2d-4a3e-9fc9-d037ae6a56d7 kind: Event lastTimestamp: "2026-07-02T19:01:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-208dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:01:22Z" name: error-404-isvc-208dc.18be8e39f49508c0 namespace: kserve-ci-e2e-test resourceVersion: "25771" uid: 141e1a08-6291-4fa3-8098-a8b8ad73f355 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-208dc namespace: kserve-ci-e2e-test resourceVersion: "26456" uid: eebd7921-cc2d-4a3e-9fc9-d037ae6a56d7 kind: Event lastTimestamp: "2026-07-02T19:02:10Z" message: InferenceService [error-404-isvc-208dc] is Ready metadata: creationTimestamp: "2026-07-02T19:02:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:02:10Z" name: error-404-isvc-208dc.18be8e453c842794 namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: de5a8fbf-db13-4ae5-be11-2895bb0652d5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:51:27.251368Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: 147ea518-52f2-41bf-8426-7f55a55d04d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l to ip-10-0-140-68.ec2.internal metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:51:27Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l.18be8daf77a4cdd4 namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: d92308e5-0765-41f5-bfad-04b3e204a6c8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 147ea518-52f2-41bf-8426-7f55a55d04d5 kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7acc8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:27Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l.18be8daf8269fdc0 namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: 911fa51f-a193-490d-9911-7e2d324ee791 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: 147ea518-52f2-41bf-8426-7f55a55d04d5 kind: Event lastTimestamp: "2026-07-02T18:51:28Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:51:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:51:28Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l.18be8dafb525b5c5 namespace: kserve-ci-e2e-test resourceVersion: "20757" uid: 419fe9b3-601f-4787-9fb4-6403127602ab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 147ea518-52f2-41bf-8426-7f55a55d04d5 kind: Event lastTimestamp: "2026-07-02T18:51:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T18:51:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:28Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l.18be8dafb6556164 namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: dc0426d5-f689-4e12-9c6b-04f20fc181b6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 147ea518-52f2-41bf-8426-7f55a55d04d5 kind: Event lastTimestamp: "2026-07-02T18:51:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:51:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:28Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l.18be8dafb99f06ed namespace: kserve-ci-e2e-test resourceVersion: "20760" uid: b230b9a4-8d24-442b-a39f-687d1f9a07b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 147ea518-52f2-41bf-8426-7f55a55d04d5 kind: Event lastTimestamp: "2026-07-02T18:51:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:51:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:28Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l.18be8dafba1351af namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 6bfd54ed-4d97-421d-9e59-45f0b44c23b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 147ea518-52f2-41bf-8426-7f55a55d04d5 kind: Event lastTimestamp: "2026-07-02T18:51:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T18:51:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:28Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l.18be8dafba1dd52f namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: 98c0b78c-5490-45d6-990b-1169b5ec6f73 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 147ea518-52f2-41bf-8426-7f55a55d04d5 kind: Event lastTimestamp: "2026-07-02T18:51:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:51:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:28Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l.18be8dafbdd59cf8 namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: b7b460ad-e8b7-4aea-852a-62c60ce6e08e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 147ea518-52f2-41bf-8426-7f55a55d04d5 kind: Event lastTimestamp: "2026-07-02T18:51:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:51:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:28Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l.18be8dafbe53eaa4 namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: 954b2490-1a15-4452-a1e0-237c04c92398 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T18:51:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 147ea518-52f2-41bf-8426-7f55a55d04d5 kind: Event lastTimestamp: "2026-07-02T19:00:45Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:45Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l.18be8dafe2bc1c90 namespace: kserve-ci-e2e-test resourceVersion: "25197" uid: 742112c9-ce46-4cfd-9c82-dc2965009700 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 147ea518-52f2-41bf-8426-7f55a55d04d5 kind: Event lastTimestamp: "2026-07-02T19:00:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:42Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l.18be8e30a633cb48 namespace: kserve-ci-e2e-test resourceVersion: "25063" uid: b9210293-99b2-4b08-a5e1-02f2ce4b97cb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 147ea518-52f2-41bf-8426-7f55a55d04d5 kind: Event lastTimestamp: "2026-07-02T19:00:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:42Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l.18be8e30a63438ed namespace: kserve-ci-e2e-test resourceVersion: "25064" uid: 08dfb159-fb88-4221-ad25-1149a11489e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l namespace: kserve-ci-e2e-test resourceVersion: "20723" uid: 147ea518-52f2-41bf-8426-7f55a55d04d5 kind: Event lastTimestamp: "2026-07-02T19:00:45Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T19:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:45Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l.18be8e31572163b1 namespace: kserve-ci-e2e-test resourceVersion: "25194" uid: 088cf837-d44e-43ad-8daf-870b4e451fc1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7acc8-predictor-7cf8fdd8fb namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 643a05f2-d004-416e-bbcc-7a3733a6d7b7 kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: 'Created pod: error-404-isvc-7acc8-predictor-7cf8fdd8fb-hk57l' metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:27Z" name: error-404-isvc-7acc8-predictor-7cf8fdd8fb.18be8daf773a72d1 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 4d5cb7f5-8d38-40a1-ba01-fc85b8c3b970 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7acc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: 44b85e92-93e4-4e78-a2a2-ea0acaa55ce1 kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: Scaled up replica set error-404-isvc-7acc8-predictor-7cf8fdd8fb from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:27Z" name: error-404-isvc-7acc8-predictor.18be8daf756ffc6a namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: e9586d8d-7cdf-4eb2-82fb-f47d95390d78 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:51:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7acc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20705" uid: 779c69b7-5fbd-4003-b0f2-c40c3e8eb153 kind: Event lastTimestamp: "2026-07-02T18:51:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:57Z" name: error-404-isvc-7acc8-predictor.18be8db2f421d3c8 namespace: kserve-ci-e2e-test resourceVersion: "21060" uid: 1a05720f-f59e-402b-b825-7af589264c03 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:51:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7acc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20705" uid: 779c69b7-5fbd-4003-b0f2-c40c3e8eb153 kind: Event lastTimestamp: "2026-07-02T18:51:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:57Z" name: error-404-isvc-7acc8-predictor.18be8db2f4877343 namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: bf3a6241-5f05-4c7c-9932-c962561c7155 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:52:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7acc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: 779c69b7-5fbd-4003-b0f2-c40c3e8eb153 kind: Event lastTimestamp: "2026-07-02T18:52:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:42Z" name: error-404-isvc-7acc8-predictor.18be8db9f1f9191a namespace: kserve-ci-e2e-test resourceVersion: "21626" uid: be3677ba-b972-4704-837e-5c09217ba257 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:52:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7acc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20895" uid: 779c69b7-5fbd-4003-b0f2-c40c3e8eb153 kind: Event lastTimestamp: "2026-07-02T18:52:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:42Z" name: error-404-isvc-7acc8-predictor.18be8db9f259f491 namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: f8868383-c3e7-4a5b-a44c-2fc84fabd79e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "20676" uid: b6d7a5ea-8f05-453c-8bf0-d990f4f2010c kind: Event lastTimestamp: "2026-07-02T18:52:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:52:15Z" name: error-404-isvc-7acc8.18be8daf75ae3379 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: af541a75-8da1-46d4-93a9-3bbd0ace4450 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "20676" uid: b6d7a5ea-8f05-453c-8bf0-d990f4f2010c kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: 'Failed to update status for InferenceService "error-404-isvc-7acc8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7acc8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:27Z" name: error-404-isvc-7acc8.18be8daf760fafd5 namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: 0d521b12-938e-41a3-9bb4-7e2591b02490 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "20676" uid: b6d7a5ea-8f05-453c-8bf0-d990f4f2010c kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7acc8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:27Z" name: error-404-isvc-7acc8.18be8daf761000f0 namespace: kserve-ci-e2e-test resourceVersion: "20719" uid: 5e099bc7-53d8-41ca-9e4a-8fc59a4731f7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "21357" uid: b6d7a5ea-8f05-453c-8bf0-d990f4f2010c kind: Event lastTimestamp: "2026-07-02T18:52:15Z" message: InferenceService [error-404-isvc-7acc8] is Ready metadata: creationTimestamp: "2026-07-02T18:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:52:15Z" name: error-404-isvc-7acc8.18be8dba9daf3c24 namespace: kserve-ci-e2e-test resourceVersion: "21358" uid: 6af36230-e8bb-453a-9d87-f4084f520428 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:50:56.806634Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf namespace: kserve-ci-e2e-test resourceVersion: "20245" uid: ad7220bc-f3d6-40c9-830c-35024c9c3b76 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf to ip-10-0-140-68.ec2.internal metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:50:56Z" name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf.18be8da860fef3fa namespace: kserve-ci-e2e-test resourceVersion: "20249" uid: c9d946e2-d0cc-4bd3-a1a9-9a4ed47a96e5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: ad7220bc-f3d6-40c9-830c-35024c9c3b76 kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-94e8c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:56Z" name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf.18be8da86b501459 namespace: kserve-ci-e2e-test resourceVersion: "20278" uid: 2fa45aee-d799-45cc-8c4d-53c45c459254 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf namespace: kserve-ci-e2e-test resourceVersion: "20254" uid: ad7220bc-f3d6-40c9-830c-35024c9c3b76 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:50:57Z" name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf.18be8da89e7e119b namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: f0882863-9133-40ca-bdc0-f711db9b4c2a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: ad7220bc-f3d6-40c9-830c-35024c9c3b76 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:57Z" name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf.18be8da89fb0dcf1 namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: 9a7d774b-a12d-4aa4-b159-9dd04e4ef52c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: ad7220bc-f3d6-40c9-830c-35024c9c3b76 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:57Z" name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf.18be8da8a2e3a028 namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: 79547c84-0f7b-4957-b61f-ba4c0f45da41 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: ad7220bc-f3d6-40c9-830c-35024c9c3b76 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:57Z" name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf.18be8da8a3a09c05 namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 53ae0944-66e4-4e2b-91c3-e1d027a4d621 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: ad7220bc-f3d6-40c9-830c-35024c9c3b76 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:57Z" name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf.18be8da8a3ab0718 namespace: kserve-ci-e2e-test resourceVersion: "20305" uid: 0dfe0274-2041-4e6a-a74d-e4e31a2654e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: ad7220bc-f3d6-40c9-830c-35024c9c3b76 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:57Z" name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf.18be8da8a6df0914 namespace: kserve-ci-e2e-test resourceVersion: "20306" uid: 0683e7fa-d8b9-444c-9dac-381f11f8e376 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: ad7220bc-f3d6-40c9-830c-35024c9c3b76 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:57Z" name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf.18be8da8a7512f80 namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: 979d9b67-c8b5-400e-b275-9881f16efbec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:50:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: ad7220bc-f3d6-40c9-830c-35024c9c3b76 kind: Event lastTimestamp: "2026-07-02T18:51:35Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:35Z" name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf.18be8da91ae02e97 namespace: kserve-ci-e2e-test resourceVersion: "20840" uid: 1bbc1a94-2884-43e5-a859-7ef1998ec144 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: ad7220bc-f3d6-40c9-830c-35024c9c3b76 kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:07Z" name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf.18be8db8c15b5733 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 3f842dbf-5930-4056-8ab6-1b6440f9918e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: ad7220bc-f3d6-40c9-830c-35024c9c3b76 kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:07Z" name: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf.18be8db8c15bad5f namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 6143b963-9435-4cd9-8fa3-3dbe800ab2af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-94e8c-predictor-5d54994ccd namespace: kserve-ci-e2e-test resourceVersion: "20238" uid: 9648c57a-8908-48f7-98c7-31a5af764b6c kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: 'Created pod: error-404-isvc-94e8c-predictor-5d54994ccd-dlmkf' metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:50:56Z" name: error-404-isvc-94e8c-predictor-5d54994ccd.18be8da860795abf namespace: kserve-ci-e2e-test resourceVersion: "20246" uid: 9fc0e3a2-4b48-4262-9a76-641a8a9d7019 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-94e8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20235" uid: 49b02e73-5d68-4621-b256-6d49198f3b9d kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: Scaled up replica set error-404-isvc-94e8c-predictor-5d54994ccd from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:50:56Z" name: error-404-isvc-94e8c-predictor.18be8da85f0c1d74 namespace: kserve-ci-e2e-test resourceVersion: "20239" uid: 66659592-cd50-4df0-9576-151f370acb3d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:51:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-94e8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20237" uid: 2f54efe6-dd51-4877-97ba-5d763bd66c91 kind: Event lastTimestamp: "2026-07-02T18:51:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:26Z" name: error-404-isvc-94e8c-predictor.18be8dabdd78a201 namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 159572c6-2384-499b-a415-1452e66b1a5a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:51:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-94e8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20237" uid: 2f54efe6-dd51-4877-97ba-5d763bd66c91 kind: Event lastTimestamp: "2026-07-02T18:51:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:26Z" name: error-404-isvc-94e8c-predictor.18be8dabddd479e9 namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 846b6899-44f3-4086-979f-acc5576737aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:51:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-94e8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: 2f54efe6-dd51-4877-97ba-5d763bd66c91 kind: Event lastTimestamp: "2026-07-02T18:51:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:56Z" name: error-404-isvc-94e8c-predictor.18be8db2db198cc0 namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: 557da861-d76c-4ba0-adc9-fb44317add92 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:51:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-94e8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: 2f54efe6-dd51-4877-97ba-5d763bd66c91 kind: Event lastTimestamp: "2026-07-02T18:51:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:56Z" name: error-404-isvc-94e8c-predictor.18be8db2db7b2777 namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 372eb753-b125-4805-993b-dd35d9c8dc57 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-94e8c namespace: kserve-ci-e2e-test resourceVersion: "20220" uid: 62ca4eef-cd5a-4858-a2d1-6812ae5f43af kind: Event lastTimestamp: "2026-07-02T18:51:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:46Z" name: error-404-isvc-94e8c.18be8da85f12f18f namespace: kserve-ci-e2e-test resourceVersion: "20943" uid: 9e81cee9-01a0-477c-8d2d-0342469b1c0b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-94e8c namespace: kserve-ci-e2e-test resourceVersion: "20220" uid: 62ca4eef-cd5a-4858-a2d1-6812ae5f43af kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-94e8c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-94e8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:50:56Z" name: error-404-isvc-94e8c.18be8da85f85a3b6 namespace: kserve-ci-e2e-test resourceVersion: "20242" uid: 71fc422c-281f-4021-a1f4-75175f4a02fc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-94e8c namespace: kserve-ci-e2e-test resourceVersion: "20220" uid: 62ca4eef-cd5a-4858-a2d1-6812ae5f43af kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-94e8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:50:56Z" name: error-404-isvc-94e8c.18be8da85f85fd5e namespace: kserve-ci-e2e-test resourceVersion: "20244" uid: 843546cf-ceb9-4a02-a83d-8715d7a1a697 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-94e8c namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: 62ca4eef-cd5a-4858-a2d1-6812ae5f43af kind: Event lastTimestamp: "2026-07-02T18:51:46Z" message: InferenceService [error-404-isvc-94e8c] is Ready metadata: creationTimestamp: "2026-07-02T18:51:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:46Z" name: error-404-isvc-94e8c.18be8db3d5ed309e namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: c6be16e0-4f6b-4f82-a2a1-f3bba2a6c37e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:40:51.982039Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: d2698acf-ea23-4665-925e-de1859ba6370 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn to ip-10-0-140-68.ec2.internal metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:40:51Z" name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn.18be8d1b8ea41d9b namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: a0b3b7ca-1f46-443a-84c6-4670ed8d34de reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: d2698acf-ea23-4665-925e-de1859ba6370 kind: Event lastTimestamp: "2026-07-02T18:40:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a5373-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T18:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:52Z" name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn.18be8d1b9a7fec01 namespace: kserve-ci-e2e-test resourceVersion: "15368" uid: cf56cebf-4fea-4313-a61d-38b97b933b54 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:53Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: d2698acf-ea23-4665-925e-de1859ba6370 kind: Event lastTimestamp: "2026-07-02T18:40:53Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:40:53Z" name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn.18be8d1bcc31491c namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: 70a17421-c89b-4e2a-bb5f-44a946160fc2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: d2698acf-ea23-4665-925e-de1859ba6370 kind: Event lastTimestamp: "2026-07-02T18:40:53Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T18:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:53Z" name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn.18be8d1bcd89abf6 namespace: kserve-ci-e2e-test resourceVersion: "15389" uid: cc5ac190-2ccd-4980-881c-bc9c06638802 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: d2698acf-ea23-4665-925e-de1859ba6370 kind: Event lastTimestamp: "2026-07-02T18:40:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:53Z" name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn.18be8d1bd19cca03 namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: c201a290-d3db-40e4-b43e-e999fd49818f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: d2698acf-ea23-4665-925e-de1859ba6370 kind: Event lastTimestamp: "2026-07-02T18:40:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:53Z" name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn.18be8d1bd217d81a namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: f5dfbab8-eefd-4d6e-8133-b7cc3eabf995 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: d2698acf-ea23-4665-925e-de1859ba6370 kind: Event lastTimestamp: "2026-07-02T18:40:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T18:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:53Z" name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn.18be8d1bd2229edf namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 82e913d1-4f51-4c93-a9a7-118d58483a05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: d2698acf-ea23-4665-925e-de1859ba6370 kind: Event lastTimestamp: "2026-07-02T18:40:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:53Z" name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn.18be8d1bd5718e02 namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: 0ec8a46b-9336-4b36-8b98-f7666983743d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: d2698acf-ea23-4665-925e-de1859ba6370 kind: Event lastTimestamp: "2026-07-02T18:40:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:53Z" name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn.18be8d1bd5e38a33 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: 956d23d6-155b-435e-b310-c14bc6f71554 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:40:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: d2698acf-ea23-4665-925e-de1859ba6370 kind: Event lastTimestamp: "2026-07-02T18:41:30Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:30Z" name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn.18be8d1c2430802f namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: 6821bcbe-0a1b-4522-9d70-ffe3fdc7aefd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: d2698acf-ea23-4665-925e-de1859ba6370 kind: Event lastTimestamp: "2026-07-02T18:50:17Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T18:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:17Z" name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn.18be8d9f1d9c0403 namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 337bc1a7-6bf0-4535-a531-dd97a83a73bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn namespace: kserve-ci-e2e-test resourceVersion: "15333" uid: d2698acf-ea23-4665-925e-de1859ba6370 kind: Event lastTimestamp: "2026-07-02T18:50:17Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:17Z" name: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn.18be8d9f1d9c83a6 namespace: kserve-ci-e2e-test resourceVersion: "19825" uid: 730f0709-21aa-4841-9552-63202950286f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a5373-predictor-7b4cdc944b namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: ca60285c-55d9-4cc7-af02-61798ad72f52 kind: Event lastTimestamp: "2026-07-02T18:40:51Z" message: 'Created pod: error-404-isvc-a5373-predictor-7b4cdc944b-lq8zn' metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:51Z" name: error-404-isvc-a5373-predictor-7b4cdc944b.18be8d1b8e3af2ea namespace: kserve-ci-e2e-test resourceVersion: "15332" uid: 7bdb2814-29c6-48b1-b656-eeaafab6a375 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a5373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 7f02bdd6-834c-4d33-81cb-bfc2cf221b48 kind: Event lastTimestamp: "2026-07-02T18:40:51Z" message: Scaled up replica set error-404-isvc-a5373-predictor-7b4cdc944b from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:51Z" name: error-404-isvc-a5373-predictor.18be8d1b8cb4a0f9 namespace: kserve-ci-e2e-test resourceVersion: "15324" uid: 68b6b77b-d3bf-4181-a846-ea71153b12a3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:41:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a5373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: bd859ef1-e47f-498e-bb5e-6614fba27ba4 kind: Event lastTimestamp: "2026-07-02T18:41:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:41:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:21Z" name: error-404-isvc-a5373-predictor.18be8d1f0b3579c5 namespace: kserve-ci-e2e-test resourceVersion: "15672" uid: bc24c1db-8751-4c9f-be87-0a402f6e3996 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:41:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a5373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: bd859ef1-e47f-498e-bb5e-6614fba27ba4 kind: Event lastTimestamp: "2026-07-02T18:41:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:41:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:21Z" name: error-404-isvc-a5373-predictor.18be8d1f0b974f34 namespace: kserve-ci-e2e-test resourceVersion: "15673" uid: dd81ba85-1667-4219-b4db-f6c83b94b42c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:41:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a5373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15551" uid: bd859ef1-e47f-498e-bb5e-6614fba27ba4 kind: Event lastTimestamp: "2026-07-02T18:41:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:41:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:52Z" name: error-404-isvc-a5373-predictor.18be8d260913c66c namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: cdc0aaf8-f4ad-4a3e-9fe0-e3895179935b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:41:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a5373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15551" uid: bd859ef1-e47f-498e-bb5e-6614fba27ba4 kind: Event lastTimestamp: "2026-07-02T18:41:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:41:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:52Z" name: error-404-isvc-a5373-predictor.18be8d26097df9cd namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: 2ac25304-1640-410b-add4-dca539ecf0d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:40:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a5373 namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: b96faf30-0102-459f-96e1-f4666a3721e3 kind: Event lastTimestamp: "2026-07-02T18:41:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:41:40Z" name: error-404-isvc-a5373.18be8d1b8cbfc5f7 namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: 29385410-ec21-45a3-8b38-517406dd3086 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a5373 namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: b96faf30-0102-459f-96e1-f4666a3721e3 kind: Event lastTimestamp: "2026-07-02T18:40:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-a5373": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a5373": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:40:51Z" name: error-404-isvc-a5373.18be8d1b8d23e2cf namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: af24ac67-ae98-4863-b9d7-a295653bcf81 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a5373 namespace: kserve-ci-e2e-test resourceVersion: "15290" uid: b96faf30-0102-459f-96e1-f4666a3721e3 kind: Event lastTimestamp: "2026-07-02T18:40:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a5373": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:40:51Z" name: error-404-isvc-a5373.18be8d1b8d244327 namespace: kserve-ci-e2e-test resourceVersion: "15329" uid: c21a05eb-47e9-418b-80d2-db0fa45b9c7f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a5373 namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: b96faf30-0102-459f-96e1-f4666a3721e3 kind: Event lastTimestamp: "2026-07-02T18:41:40Z" message: InferenceService [error-404-isvc-a5373] is Ready metadata: creationTimestamp: "2026-07-02T18:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:41:40Z" name: error-404-isvc-a5373.18be8d26df3dc24a namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: 219eaf45-1abd-4ee3-bd4f-12e627395d2f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:52:07.075736Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 2838a1ec-8335-4af1-8956-fd80ed486104 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ae135-predictor-5dfb866474-tfrnf to ip-10-0-140-68.ec2.internal metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:52:07Z" name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf.18be8db8bd5c7b22 namespace: kserve-ci-e2e-test resourceVersion: "21224" uid: 31676dcf-1c3e-464e-a202-06b7958a5702 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 2838a1ec-8335-4af1-8956-fd80ed486104 kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-ae135-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:07Z" name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf.18be8db8c8fdb5e3 namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: 841f5051-ca22-4471-8212-5117ee0d481d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:08Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: 2838a1ec-8335-4af1-8956-fd80ed486104 kind: Event lastTimestamp: "2026-07-02T18:52:08Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:52:08Z" name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf.18be8db8fb3be4fa namespace: kserve-ci-e2e-test resourceVersion: "21268" uid: c59a8302-22ca-4480-ae7b-3ca783c17760 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 2838a1ec-8335-4af1-8956-fd80ed486104 kind: Event lastTimestamp: "2026-07-02T18:52:08Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T18:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:08Z" name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf.18be8db8fd15e96e namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: 8996c229-79dc-4207-b2b4-4003f568e941 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 2838a1ec-8335-4af1-8956-fd80ed486104 kind: Event lastTimestamp: "2026-07-02T18:52:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:08Z" name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf.18be8db90082d0c9 namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 30196a15-7f49-46d9-b397-3f72b2a717b6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 2838a1ec-8335-4af1-8956-fd80ed486104 kind: Event lastTimestamp: "2026-07-02T18:52:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:08Z" name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf.18be8db9013196a6 namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 3755c4eb-a9cc-4890-9c10-9b01a6ef886d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 2838a1ec-8335-4af1-8956-fd80ed486104 kind: Event lastTimestamp: "2026-07-02T18:52:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T18:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:08Z" name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf.18be8db9013caef1 namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: 774928fc-418c-4024-aa6d-8aac4cd508c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 2838a1ec-8335-4af1-8956-fd80ed486104 kind: Event lastTimestamp: "2026-07-02T18:52:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:08Z" name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf.18be8db904873904 namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: d1084607-e4e4-4a55-8d38-8e72b22f2f78 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 2838a1ec-8335-4af1-8956-fd80ed486104 kind: Event lastTimestamp: "2026-07-02T18:52:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:08Z" name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf.18be8db904f6d3bc namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: e24e9546-6762-4e1d-b685-1836fcbdf96d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T18:52:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 2838a1ec-8335-4af1-8956-fd80ed486104 kind: Event lastTimestamp: "2026-07-02T19:01:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:52:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:25Z" name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf.18be8db93d003703 namespace: kserve-ci-e2e-test resourceVersion: "25850" uid: 07d01bed-b8ca-4343-8c8d-fdc36d75f716 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 2838a1ec-8335-4af1-8956-fd80ed486104 kind: Event lastTimestamp: "2026-07-02T19:01:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T19:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:21Z" name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf.18be8e39ebebcbe1 namespace: kserve-ci-e2e-test resourceVersion: "25709" uid: 40d153b6-f363-4c11-bd9c-91c95457f64f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 2838a1ec-8335-4af1-8956-fd80ed486104 kind: Event lastTimestamp: "2026-07-02T19:01:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:21Z" name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf.18be8e39ebec4664 namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: 39e0727a-6731-4574-ab25-75ed422c8fdf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 2838a1ec-8335-4af1-8956-fd80ed486104 kind: Event lastTimestamp: "2026-07-02T19:01:25Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": dial tcp 10.132.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T19:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:25Z" name: error-404-isvc-ae135-predictor-5dfb866474-tfrnf.18be8e3ab16446db namespace: kserve-ci-e2e-test resourceVersion: "25849" uid: 8ee78034-fc55-4765-91d9-c6d5c60464c0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ae135-predictor-5dfb866474 namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 8fdac4f1-2130-40ce-8d73-de756c25694f kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: 'Created pod: error-404-isvc-ae135-predictor-5dfb866474-tfrnf' metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:07Z" name: error-404-isvc-ae135-predictor-5dfb866474.18be8db8bca6e0c7 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 8b2e49a3-93b0-4c84-86f6-35c7245a1ca2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ae135-predictor namespace: kserve-ci-e2e-test resourceVersion: "21208" uid: e47e3c91-d60b-488d-b0fb-7644ac2f493c kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: Scaled up replica set error-404-isvc-ae135-predictor-5dfb866474 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:07Z" name: error-404-isvc-ae135-predictor.18be8db8bb4a2c8f namespace: kserve-ci-e2e-test resourceVersion: "21213" uid: 019af1c1-47d9-488c-8dac-b7c7d5477de9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:52:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ae135-predictor namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: cbac9293-0bc8-4ef2-a715-db51a5a495c3 kind: Event lastTimestamp: "2026-07-02T18:52:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:37Z" name: error-404-isvc-ae135-predictor.18be8dbc39ebbe30 namespace: kserve-ci-e2e-test resourceVersion: "21584" uid: 18deb45f-48e1-4bde-a8fe-f07a29bcf60c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:52:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ae135-predictor namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: cbac9293-0bc8-4ef2-a715-db51a5a495c3 kind: Event lastTimestamp: "2026-07-02T18:52:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:37Z" name: error-404-isvc-ae135-predictor.18be8dbc3a2c004d namespace: kserve-ci-e2e-test resourceVersion: "21585" uid: 1e874532-0509-4826-b71b-0f72d916abdc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:52:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ae135-predictor namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: cbac9293-0bc8-4ef2-a715-db51a5a495c3 kind: Event lastTimestamp: "2026-07-02T18:53:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:53:07Z" name: error-404-isvc-ae135-predictor.18be8dc337b91753 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: bef70c95-1683-4fe6-b5b2-b7cac33c0156 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:52:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ae135-predictor namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: cbac9293-0bc8-4ef2-a715-db51a5a495c3 kind: Event lastTimestamp: "2026-07-02T18:53:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:53:07Z" name: error-404-isvc-ae135-predictor.18be8dc33817765e namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 5c4c318a-5213-4995-abd4-ac60d00c875e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ae135 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 25b63101-1360-4647-b3f2-17c86568a1ae kind: Event lastTimestamp: "2026-07-02T18:52:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:52:55Z" name: error-404-isvc-ae135.18be8db8bb99921d namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: 618810c9-62e8-4c82-9363-ec084c9c9c94 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ae135 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 25b63101-1360-4647-b3f2-17c86568a1ae kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: 'Failed to update status for InferenceService "error-404-isvc-ae135": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ae135": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:52:07Z" name: error-404-isvc-ae135.18be8db8bbe75c9b namespace: kserve-ci-e2e-test resourceVersion: "21216" uid: fb7ae883-338c-44b1-8da8-ff7d087490a7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ae135 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 25b63101-1360-4647-b3f2-17c86568a1ae kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ae135": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:52:07Z" name: error-404-isvc-ae135.18be8db8bbe7ad34 namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: 252d1970-88dc-4314-8de9-66019ed24e75 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ae135 namespace: kserve-ci-e2e-test resourceVersion: "21730" uid: 25b63101-1360-4647-b3f2-17c86568a1ae kind: Event lastTimestamp: "2026-07-02T18:52:55Z" message: InferenceService [error-404-isvc-ae135] is Ready metadata: creationTimestamp: "2026-07-02T18:52:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:52:55Z" name: error-404-isvc-ae135.18be8dc3f7ba38e2 namespace: kserve-ci-e2e-test resourceVersion: "21731" uid: 3c42a22c-4af8-4ee1-89fb-f9e33d5ad55e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:39:11.879304Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14230" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 to ip-10-0-140-68.ec2.internal metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:39:11Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d04400d9176 namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 7fbe5db3-f3f6-4e10-aec7-7fd9ac016ec8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14239" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: "2026-07-02T18:39:12Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:39:12Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d045a68ad2f namespace: kserve-ci-e2e-test resourceVersion: "14313" uid: ed33649c-95e3-4a0a-99eb-addcecae0c5c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: "2026-07-02T18:39:12Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1698" metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:12Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d045b9f920d namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: 3e8c56db-3ad1-496a-abca-68ec6ba72ac1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: "2026-07-02T18:39:44Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1698" in 31.922s (31.922s including waiting). Image size: 1334625882 bytes.' metadata: creationTimestamp: "2026-07-02T18:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:44Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d0bca5b8310 namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: 69c32d4c-64c1-46b1-8ed1-7cab001cd27f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: "2026-07-02T18:39:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:44Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d0bd92e9822 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 8ba129ac-6b94-4eaf-b1d3-7eb85f2956a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: "2026-07-02T18:39:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:44Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d0bda695fcd namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: e4c8abd1-22c4-4d27-ad8f-0f252aad7481 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: "2026-07-02T18:39:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T18:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:44Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d0bda785607 namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 654e6cf0-73ee-4901-9e94-0c70d88e3bca reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: "2026-07-02T18:39:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.38s (3.38s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T18:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:47Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d0ca3f4b952 namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: a34e3645-33f2-4b51-8ae6-a6348be9683e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: "2026-07-02T18:39:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:48Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d0ca9409369 namespace: kserve-ci-e2e-test resourceVersion: "14663" uid: f45fed88-9b31-436e-89b4-d1ab18bd004d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: "2026-07-02T18:39:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:48Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d0ca9b7500c namespace: kserve-ci-e2e-test resourceVersion: "14665" uid: 7104e44d-9a07-4fcf-a91d-078b5982b28a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T18:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: "2026-07-02T18:40:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:54Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d0cb487a05d namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 6e749cc0-3bd8-42f6-81b8-468401f507d5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: "2026-07-02T18:40:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T18:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:52Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d1b920e9883 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 4ff7be46-4a71-4b2c-bf3c-d3e9fd38b2e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: "2026-07-02T18:40:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:52Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d1b920f00d2 namespace: kserve-ci-e2e-test resourceVersion: "15360" uid: bb6737e7-f6a0-44ae-98fb-68748c241621 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6 namespace: kserve-ci-e2e-test resourceVersion: "14232" uid: 5e792528-8683-400b-a079-aa659d1a1c5f kind: Event lastTimestamp: "2026-07-02T18:40:54Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:54Z" name: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6.18be8d1c12c0d13a namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: b8fe768a-c3bd-4806-9fde-c18e7709c4f9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bb242-predictor-7d85f5f688 namespace: kserve-ci-e2e-test resourceVersion: "14223" uid: 68824517-d9ee-49ae-82a5-98f858bcd2a6 kind: Event lastTimestamp: "2026-07-02T18:39:11Z" message: 'Created pod: error-404-isvc-bb242-predictor-7d85f5f688-cxpx6' metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:11Z" name: error-404-isvc-bb242-predictor-7d85f5f688.18be8d043fa8a84a namespace: kserve-ci-e2e-test resourceVersion: "14231" uid: 069791ab-95f8-49f9-98dd-4d7daf40e291 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bb242-predictor namespace: kserve-ci-e2e-test resourceVersion: "14220" uid: 4d1686b6-f047-4b05-93fc-c12292161840 kind: Event lastTimestamp: "2026-07-02T18:39:11Z" message: Scaled up replica set error-404-isvc-bb242-predictor-7d85f5f688 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:11Z" name: error-404-isvc-bb242-predictor.18be8d043d02746f namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: 32ec0e6e-761a-4378-94a1-bbc6375d8207 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T18:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bb242-predictor namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e680818-3508-4501-9842-4e35f7823fe6 kind: Event lastTimestamp: "2026-07-02T18:40:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:11Z" name: error-404-isvc-bb242-predictor.18be8d07bb7409b4 namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: dcdc7627-c532-43dc-bed2-fec88ad6c1f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T18:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bb242-predictor namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: 8e680818-3508-4501-9842-4e35f7823fe6 kind: Event lastTimestamp: "2026-07-02T18:40:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:11Z" name: error-404-isvc-bb242-predictor.18be8d07bbd47571 namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: a9b23cca-8215-4a64-b3ea-a30e37cdeaae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:40:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bb242-predictor namespace: kserve-ci-e2e-test resourceVersion: "14476" uid: 8e680818-3508-4501-9842-4e35f7823fe6 kind: Event lastTimestamp: "2026-07-02T18:40:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:41Z" name: error-404-isvc-bb242-predictor.18be8d15b6614d53 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: d26e4c05-6fa6-4382-aa84-4f8404dccb90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:40:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bb242-predictor namespace: kserve-ci-e2e-test resourceVersion: "14476" uid: 8e680818-3508-4501-9842-4e35f7823fe6 kind: Event lastTimestamp: "2026-07-02T18:40:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:41Z" name: error-404-isvc-bb242-predictor.18be8d15b6c1fea1 namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: e757dab2-4406-4141-8dcb-d2595bff56ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bb242 namespace: kserve-ci-e2e-test resourceVersion: "14206" uid: e40c9239-8d60-428a-8f42-913197c8abde kind: Event lastTimestamp: "2026-07-02T18:40:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:40:34Z" name: error-404-isvc-bb242.18be8d043d07fc5a namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: c6734c16-fb10-439e-b7af-954d11f367f6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bb242 namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: e40c9239-8d60-428a-8f42-913197c8abde kind: Event lastTimestamp: "2026-07-02T18:40:34Z" message: InferenceService [error-404-isvc-bb242] is Ready metadata: creationTimestamp: "2026-07-02T18:40:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:40:34Z" name: error-404-isvc-bb242.18be8d176fb26541 namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: 2600546e-b0b0-416d-b2d6-5ff754b05d5e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T19:01:52.294148Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck namespace: kserve-ci-e2e-test resourceVersion: "26244" uid: 3a1f430b-1817-4fd5-aeef-7d0b4b354bef kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c41f9-predictor-78b9f9dc78-glpck to ip-10-0-140-68.ec2.internal metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T19:01:52Z" name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck.18be8e40ff18c4dd namespace: kserve-ci-e2e-test resourceVersion: "26253" uid: ba12b425-3de2-42c8-ae45-cb8191efa6d8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 3a1f430b-1817-4fd5-aeef-7d0b4b354bef kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c41f9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:52Z" name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck.18be8e410709f432 namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: 400c551b-8eb0-4cb8-aae8-1716d9f17638 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:53Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck namespace: kserve-ci-e2e-test resourceVersion: "26260" uid: 3a1f430b-1817-4fd5-aeef-7d0b4b354bef kind: Event lastTimestamp: "2026-07-02T19:01:53Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T19:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T19:01:53Z" name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck.18be8e413c25cac1 namespace: kserve-ci-e2e-test resourceVersion: "26286" uid: 1a4081bc-1ae1-4717-af7c-34c72b115f37 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 3a1f430b-1817-4fd5-aeef-7d0b4b354bef kind: Event lastTimestamp: "2026-07-02T19:01:53Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T19:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:53Z" name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck.18be8e413d7b6c2a namespace: kserve-ci-e2e-test resourceVersion: "26288" uid: d6fcd964-32cb-48c1-8377-2278d25b15e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 3a1f430b-1817-4fd5-aeef-7d0b4b354bef kind: Event lastTimestamp: "2026-07-02T19:01:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T19:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:53Z" name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck.18be8e414039283e namespace: kserve-ci-e2e-test resourceVersion: "26290" uid: d7952b1d-8d99-4c61-9700-fd2bebb727bd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 3a1f430b-1817-4fd5-aeef-7d0b4b354bef kind: Event lastTimestamp: "2026-07-02T19:01:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T19:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:53Z" name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck.18be8e4140a8fa06 namespace: kserve-ci-e2e-test resourceVersion: "26291" uid: 69b72206-28aa-4ccb-b019-ba37d650eebb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 3a1f430b-1817-4fd5-aeef-7d0b4b354bef kind: Event lastTimestamp: "2026-07-02T19:01:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T19:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:53Z" name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck.18be8e4140b31643 namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: f25f5520-b242-4eaf-847e-fe47252f47fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 3a1f430b-1817-4fd5-aeef-7d0b4b354bef kind: Event lastTimestamp: "2026-07-02T19:01:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T19:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:53Z" name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck.18be8e4143c08f5c namespace: kserve-ci-e2e-test resourceVersion: "26294" uid: 5cbfd39e-5aa7-4e4f-8031-d2e48addf2c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 3a1f430b-1817-4fd5-aeef-7d0b4b354bef kind: Event lastTimestamp: "2026-07-02T19:01:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:01:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:53Z" name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck.18be8e4144355553 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: e4628a24-b97a-4d08-978d-5694de25988c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T19:01:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 3a1f430b-1817-4fd5-aeef-7d0b4b354bef kind: Event lastTimestamp: "2026-07-02T19:02:30Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T19:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:02:30Z" name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck.18be8e41854ffec2 namespace: kserve-ci-e2e-test resourceVersion: "26662" uid: 6d9645ab-0768-465c-b0e2-2bb7477f9e56 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 3a1f430b-1817-4fd5-aeef-7d0b4b354bef kind: Event lastTimestamp: "2026-07-02T19:11:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T19:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:11:07Z" name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck.18be8ec22c76080c namespace: kserve-ci-e2e-test resourceVersion: "30345" uid: 6c355142-5c1c-4963-9a4c-9912156f7374 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 3a1f430b-1817-4fd5-aeef-7d0b4b354bef kind: Event lastTimestamp: "2026-07-02T19:11:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:11:07Z" name: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck.18be8ec22c77517d namespace: kserve-ci-e2e-test resourceVersion: "30346" uid: 66773e35-939b-4772-aa26-e184d2a0eaad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c41f9-predictor-78b9f9dc78 namespace: kserve-ci-e2e-test resourceVersion: "26230" uid: 7cbfb2fc-150d-4fdd-98c0-37c37df9461f kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: 'Created pod: error-404-isvc-c41f9-predictor-78b9f9dc78-glpck' metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:52Z" name: error-404-isvc-c41f9-predictor-78b9f9dc78.18be8e40feab4d41 namespace: kserve-ci-e2e-test resourceVersion: "26249" uid: e6ca9521-dd46-42d5-b387-1143a343e726 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c41f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26224" uid: 5ac64658-a114-45b5-8e7a-ad50b9d8456f kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: Scaled up replica set error-404-isvc-c41f9-predictor-78b9f9dc78 from 0 to 1 metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:52Z" name: error-404-isvc-c41f9-predictor.18be8e40fcdda599 namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: 1ae76dda-bd5f-4353-b9ec-975a6230bfee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:02:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c41f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26229" uid: e5b00e76-c459-4340-a8d3-5a39351ec054 kind: Event lastTimestamp: "2026-07-02T19:02:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T19:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:22Z" name: error-404-isvc-c41f9-predictor.18be8e447b887c17 namespace: kserve-ci-e2e-test resourceVersion: "26586" uid: d65b4ea1-41c2-487a-b71d-97accaaf93ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:02:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c41f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26229" uid: e5b00e76-c459-4340-a8d3-5a39351ec054 kind: Event lastTimestamp: "2026-07-02T19:02:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T19:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:22Z" name: error-404-isvc-c41f9-predictor.18be8e447beb3470 namespace: kserve-ci-e2e-test resourceVersion: "26587" uid: 02503296-43df-49c6-82f5-0f90fe4d340b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T19:02:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c41f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26426" uid: e5b00e76-c459-4340-a8d3-5a39351ec054 kind: Event lastTimestamp: "2026-07-02T19:03:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:03:07Z" name: error-404-isvc-c41f9-predictor.18be8e4b79389035 namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: 8a8fdc31-c729-4de3-babc-488e6d9b3358 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T19:02:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c41f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26426" uid: e5b00e76-c459-4340-a8d3-5a39351ec054 kind: Event lastTimestamp: "2026-07-02T19:03:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:03:07Z" name: error-404-isvc-c41f9-predictor.18be8e4b799f76c4 namespace: kserve-ci-e2e-test resourceVersion: "26999" uid: d6dadfee-c26b-472b-b0ac-26ee2e12f718 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c41f9 namespace: kserve-ci-e2e-test resourceVersion: "26211" uid: 1df4ffcc-c429-418d-9ea5-bce8e977b1f7 kind: Event lastTimestamp: "2026-07-02T19:02:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:02:40Z" name: error-404-isvc-c41f9.18be8e40fd0db1b9 namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: 3a52f403-0ea9-4d04-858c-e856d7f61282 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c41f9 namespace: kserve-ci-e2e-test resourceVersion: "26760" uid: 1df4ffcc-c429-418d-9ea5-bce8e977b1f7 kind: Event lastTimestamp: "2026-07-02T19:02:40Z" message: InferenceService [error-404-isvc-c41f9] is Ready metadata: creationTimestamp: "2026-07-02T19:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:02:40Z" name: error-404-isvc-c41f9.18be8e4c401e7a0e namespace: kserve-ci-e2e-test resourceVersion: "26761" uid: 0b83a7cd-b06c-4926-a2c2-364311c3063c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T19:00:42.048553Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c532a-predictor-78564c9984-cltvv namespace: kserve-ci-e2e-test resourceVersion: "25042" uid: 4efd3f46-02ad-4da3-bbb3-1e889542a4c4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c532a-predictor-78564c9984-cltvv to ip-10-0-140-68.ec2.internal metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T19:00:42Z" name: error-404-isvc-c532a-predictor-78564c9984-cltvv.18be8e30a4220fef namespace: kserve-ci-e2e-test resourceVersion: "25047" uid: 1139b860-2230-4f9f-a214-1477c24080c3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c532a-predictor-78564c9984-cltvv namespace: kserve-ci-e2e-test resourceVersion: "25061" uid: 4efd3f46-02ad-4da3-bbb3-1e889542a4c4 kind: Event lastTimestamp: "2026-07-02T19:00:42Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T19:00:42Z" name: error-404-isvc-c532a-predictor-78564c9984-cltvv.18be8e30bde5479b namespace: kserve-ci-e2e-test resourceVersion: "25082" uid: 9e98d60e-a773-4337-bb34-4de09835f643 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c532a-predictor-78564c9984-cltvv namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: 4efd3f46-02ad-4da3-bbb3-1e889542a4c4 kind: Event lastTimestamp: "2026-07-02T19:00:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:42Z" name: error-404-isvc-c532a-predictor-78564c9984-cltvv.18be8e30bf067324 namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 74663110-69d4-49d3-8d2a-6c494b7a70a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c532a-predictor-78564c9984-cltvv namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: 4efd3f46-02ad-4da3-bbb3-1e889542a4c4 kind: Event lastTimestamp: "2026-07-02T19:00:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:42Z" name: error-404-isvc-c532a-predictor-78564c9984-cltvv.18be8e30c24c6c01 namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: 0c6ebe88-1954-4be1-94a4-22a3dd28e630 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c532a-predictor-78564c9984-cltvv namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: 4efd3f46-02ad-4da3-bbb3-1e889542a4c4 kind: Event lastTimestamp: "2026-07-02T19:00:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:42Z" name: error-404-isvc-c532a-predictor-78564c9984-cltvv.18be8e30c2bf0312 namespace: kserve-ci-e2e-test resourceVersion: "25086" uid: 604cba8f-49e1-4bb9-94a7-76674b3d0f7c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c532a-predictor-78564c9984-cltvv namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: 4efd3f46-02ad-4da3-bbb3-1e889542a4c4 kind: Event lastTimestamp: "2026-07-02T19:00:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:42Z" name: error-404-isvc-c532a-predictor-78564c9984-cltvv.18be8e30c2c98e1e namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: e49fc074-01c5-4541-a783-4bbdbe409234 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c532a-predictor-78564c9984-cltvv namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: 4efd3f46-02ad-4da3-bbb3-1e889542a4c4 kind: Event lastTimestamp: "2026-07-02T19:00:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:42Z" name: error-404-isvc-c532a-predictor-78564c9984-cltvv.18be8e30c6ffb45d namespace: kserve-ci-e2e-test resourceVersion: "25088" uid: 6b5c9b7b-0d82-4574-aae2-f16d8ac01471 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c532a-predictor-78564c9984-cltvv namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: 4efd3f46-02ad-4da3-bbb3-1e889542a4c4 kind: Event lastTimestamp: "2026-07-02T19:00:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:42Z" name: error-404-isvc-c532a-predictor-78564c9984-cltvv.18be8e30c77e2b83 namespace: kserve-ci-e2e-test resourceVersion: "25089" uid: 83375917-8e64-4cc7-a6ff-6be2839c531b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T19:00:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c532a-predictor-78564c9984-cltvv namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: 4efd3f46-02ad-4da3-bbb3-1e889542a4c4 kind: Event lastTimestamp: "2026-07-02T19:01:19Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T19:00:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:19Z" name: error-404-isvc-c532a-predictor-78564c9984-cltvv.18be8e30eb9ce875 namespace: kserve-ci-e2e-test resourceVersion: "25659" uid: 5386d0f8-5bf0-4aa7-b77a-5d30111c063b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c532a-predictor-78564c9984-cltvv namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: 4efd3f46-02ad-4da3-bbb3-1e889542a4c4 kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:52Z" name: error-404-isvc-c532a-predictor-78564c9984-cltvv.18be8e40fe23d083 namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 310408d4-f06f-4259-9c7f-b61b67c0ede8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c532a-predictor-78564c9984-cltvv namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: 4efd3f46-02ad-4da3-bbb3-1e889542a4c4 kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:52Z" name: error-404-isvc-c532a-predictor-78564c9984-cltvv.18be8e40fe24c662 namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: bbca7eae-6a02-4666-89e9-5706598e1228 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c532a-predictor-78564c9984-cltvv namespace: kserve-ci-e2e-test resourceVersion: "25044" uid: 4efd3f46-02ad-4da3-bbb3-1e889542a4c4 kind: Event lastTimestamp: "2026-07-02T19:01:54Z" message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": dial tcp 10.132.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T19:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:54Z" name: error-404-isvc-c532a-predictor-78564c9984-cltvv.18be8e4173badacb namespace: kserve-ci-e2e-test resourceVersion: "26306" uid: 89d5412d-52fa-40ee-9557-b733350623dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c532a-predictor-78564c9984 namespace: kserve-ci-e2e-test resourceVersion: "25035" uid: c3e66e4f-c9b2-424e-98e4-0a5566a40e67 kind: Event lastTimestamp: "2026-07-02T19:00:42Z" message: 'Created pod: error-404-isvc-c532a-predictor-78564c9984-cltvv' metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:00:42Z" name: error-404-isvc-c532a-predictor-78564c9984.18be8e30a3bc891a namespace: kserve-ci-e2e-test resourceVersion: "25043" uid: 0f5a55cc-6dc1-44e5-ae41-034a53a7c245 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c532a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25033" uid: b4528cf0-c4d5-4b93-b9d0-69316f93612a kind: Event lastTimestamp: "2026-07-02T19:00:42Z" message: Scaled up replica set error-404-isvc-c532a-predictor-78564c9984 from 0 to 1 metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:00:42Z" name: error-404-isvc-c532a-predictor.18be8e30a2220b2b namespace: kserve-ci-e2e-test resourceVersion: "25036" uid: 1ed08633-17e5-4f24-8fa6-ae1e8e0abc9f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:00:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c532a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25034" uid: dfc5feeb-d1fc-45cf-8cf7-0f94c94992eb kind: Event lastTimestamp: "2026-07-02T19:01:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T19:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:12Z" name: error-404-isvc-c532a-predictor.18be8e34208a6d15 namespace: kserve-ci-e2e-test resourceVersion: "25601" uid: 9e934eac-bc35-485d-bdb7-5aa27fdb5dd4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:00:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c532a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25034" uid: dfc5feeb-d1fc-45cf-8cf7-0f94c94992eb kind: Event lastTimestamp: "2026-07-02T19:01:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T19:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:12Z" name: error-404-isvc-c532a-predictor.18be8e3420e5ae06 namespace: kserve-ci-e2e-test resourceVersion: "25602" uid: 43e46b42-784b-4cc8-9450-631b43050033 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:01:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c532a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25486" uid: dfc5feeb-d1fc-45cf-8cf7-0f94c94992eb kind: Event lastTimestamp: "2026-07-02T19:01:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:42Z" name: error-404-isvc-c532a-predictor.18be8e3b1e9a0bf4 namespace: kserve-ci-e2e-test resourceVersion: "26056" uid: 13cba519-0ee7-4cda-af27-cbf2e9995fd5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:01:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c532a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25486" uid: dfc5feeb-d1fc-45cf-8cf7-0f94c94992eb kind: Event lastTimestamp: "2026-07-02T19:01:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:42Z" name: error-404-isvc-c532a-predictor.18be8e3b1f091913 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: 8ddad0eb-6c78-4082-bda0-a41538ea41f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T19:00:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c532a namespace: kserve-ci-e2e-test resourceVersion: "25015" uid: f7bb7a33-1080-41ec-b1b9-d8a1c57cfec1 kind: Event lastTimestamp: "2026-07-02T19:01:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:01:29Z" name: error-404-isvc-c532a.18be8e30a22d28a6 namespace: kserve-ci-e2e-test resourceVersion: "25903" uid: f81b80c4-bbc6-462c-aa4d-bf59d10d08e8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c532a namespace: kserve-ci-e2e-test resourceVersion: "25901" uid: f7bb7a33-1080-41ec-b1b9-d8a1c57cfec1 kind: Event lastTimestamp: "2026-07-02T19:01:29Z" message: InferenceService [error-404-isvc-c532a] is Ready metadata: creationTimestamp: "2026-07-02T19:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:01:29Z" name: error-404-isvc-c532a.18be8e3ba693cac1 namespace: kserve-ci-e2e-test resourceVersion: "25902" uid: 6f53fb0f-9563-4f83-8a0e-3e7759cd1b0d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:50:16.965186Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw namespace: kserve-ci-e2e-test resourceVersion: "19800" uid: a65e808b-dd46-48a8-a950-a5f65fd8c002 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw to ip-10-0-140-68.ec2.internal metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:50:16Z" name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw.18be8d9f1a42cfb7 namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: 90fd7b5a-e74c-4583-824e-fa2ba9005222 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: a65e808b-dd46-48a8-a950-a5f65fd8c002 kind: Event lastTimestamp: "2026-07-02T18:50:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e22cf-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T18:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:17Z" name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw.18be8d9f24fe49a4 namespace: kserve-ci-e2e-test resourceVersion: "19831" uid: 71e3fab0-2236-43c8-a8c0-399a91fab486 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw namespace: kserve-ci-e2e-test resourceVersion: "19815" uid: a65e808b-dd46-48a8-a950-a5f65fd8c002 kind: Event lastTimestamp: "2026-07-02T18:50:17Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:50:17Z" name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw.18be8d9f5785b85e namespace: kserve-ci-e2e-test resourceVersion: "19854" uid: 5bcbee08-dadb-4ac7-841e-7c16aeafb0fa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: a65e808b-dd46-48a8-a950-a5f65fd8c002 kind: Event lastTimestamp: "2026-07-02T18:50:18Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T18:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:18Z" name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw.18be8d9f58b9d513 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: 15c552b3-506b-4999-9d54-0c075edf4bb6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: a65e808b-dd46-48a8-a950-a5f65fd8c002 kind: Event lastTimestamp: "2026-07-02T18:50:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:18Z" name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw.18be8d9f5baa804e namespace: kserve-ci-e2e-test resourceVersion: "19858" uid: a95879ac-d08d-473e-b5ee-e57aeb8683f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: a65e808b-dd46-48a8-a950-a5f65fd8c002 kind: Event lastTimestamp: "2026-07-02T18:50:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:18Z" name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw.18be8d9f5c1c5da1 namespace: kserve-ci-e2e-test resourceVersion: "19859" uid: 523c199f-f1c8-414c-ad09-7a45de4cfbc8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: a65e808b-dd46-48a8-a950-a5f65fd8c002 kind: Event lastTimestamp: "2026-07-02T18:50:18Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T18:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:18Z" name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw.18be8d9f5c2606a6 namespace: kserve-ci-e2e-test resourceVersion: "19860" uid: 61f404ea-1ae0-4809-9cff-8e7e6c3c5c9a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: a65e808b-dd46-48a8-a950-a5f65fd8c002 kind: Event lastTimestamp: "2026-07-02T18:50:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:18Z" name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw.18be8d9f60014b61 namespace: kserve-ci-e2e-test resourceVersion: "19861" uid: 2c959a5a-7ea7-4c85-a054-bca1db5f7ad5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: a65e808b-dd46-48a8-a950-a5f65fd8c002 kind: Event lastTimestamp: "2026-07-02T18:50:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:18Z" name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw.18be8d9f60792999 namespace: kserve-ci-e2e-test resourceVersion: "19862" uid: 983ac0a5-616b-4217-8b1e-3531dd60e5a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:50:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: a65e808b-dd46-48a8-a950-a5f65fd8c002 kind: Event lastTimestamp: "2026-07-02T18:50:55Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:55Z" name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw.18be8d9fc10ffe89 namespace: kserve-ci-e2e-test resourceVersion: "20160" uid: 0430ca4f-a5e5-4a41-b970-117e51c23182 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: a65e808b-dd46-48a8-a950-a5f65fd8c002 kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:27Z" name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw.18be8daf760dbf04 namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: eb319a7a-51f7-42a6-bd90-d7fea139b1a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: a65e808b-dd46-48a8-a950-a5f65fd8c002 kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:27Z" name: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw.18be8daf760e256e namespace: kserve-ci-e2e-test resourceVersion: "20717" uid: 944e3562-e5d7-49b0-9745-488dfe00a8dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e22cf-predictor-77d4dd487b namespace: kserve-ci-e2e-test resourceVersion: "19791" uid: d963b7ca-2440-494a-a5f1-0bf9752dd322 kind: Event lastTimestamp: "2026-07-02T18:50:16Z" message: 'Created pod: error-404-isvc-e22cf-predictor-77d4dd487b-cdhfw' metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:50:16Z" name: error-404-isvc-e22cf-predictor-77d4dd487b.18be8d9f19c08827 namespace: kserve-ci-e2e-test resourceVersion: "19802" uid: 271da489-f871-439d-bcb7-7fc22b8fa145 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e22cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: 82aa5626-9151-4e7b-abba-9dcf10ace07b kind: Event lastTimestamp: "2026-07-02T18:50:16Z" message: Scaled up replica set error-404-isvc-e22cf-predictor-77d4dd487b from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:50:16Z" name: error-404-isvc-e22cf-predictor.18be8d9f18105376 namespace: kserve-ci-e2e-test resourceVersion: "19793" uid: 3bec6be5-0b6b-4989-9584-611ad78406b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e22cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: 36181a79-ad66-46bb-ac77-7464f2e0f253 kind: Event lastTimestamp: "2026-07-02T18:50:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:50:31Z" name: error-404-isvc-e22cf-predictor.18be8da296bd4686 namespace: kserve-ci-e2e-test resourceVersion: "19981" uid: e9bb0c08-7347-4469-b43f-8a5dc852ad46 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e22cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: 36181a79-ad66-46bb-ac77-7464f2e0f253 kind: Event lastTimestamp: "2026-07-02T18:50:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:50:31Z" name: error-404-isvc-e22cf-predictor.18be8da2971fb3b7 namespace: kserve-ci-e2e-test resourceVersion: "19983" uid: d41c8e0f-a2e8-44f4-8435-861487f4446c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:50:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e22cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19982" uid: 36181a79-ad66-46bb-ac77-7464f2e0f253 kind: Event lastTimestamp: "2026-07-02T18:51:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:16Z" name: error-404-isvc-e22cf-predictor.18be8da615bf84a4 namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: 02877f0d-2cd4-47bf-b9f3-e733c2d1d137 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:50:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e22cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19982" uid: 36181a79-ad66-46bb-ac77-7464f2e0f253 kind: Event lastTimestamp: "2026-07-02T18:51:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:16Z" name: error-404-isvc-e22cf-predictor.18be8da61621afed namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: 36178b75-e0ed-4e26-923a-2b4dccacf080 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:50:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e22cf namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: 365ff1d9-ec72-4d63-8281-a7089f1230ba kind: Event lastTimestamp: "2026-07-02T18:51:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:05Z" name: error-404-isvc-e22cf.18be8d9f183dde04 namespace: kserve-ci-e2e-test resourceVersion: "20409" uid: 85a05393-0e28-46e6-abc7-aa2a42e487dc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e22cf namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: 365ff1d9-ec72-4d63-8281-a7089f1230ba kind: Event lastTimestamp: "2026-07-02T18:50:16Z" message: 'Failed to update status for InferenceService "error-404-isvc-e22cf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e22cf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:50:16Z" name: error-404-isvc-e22cf.18be8d9f1896e843 namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: 3d26ebf2-6980-4f74-ae48-4a8b6106d0a0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e22cf namespace: kserve-ci-e2e-test resourceVersion: "19757" uid: 365ff1d9-ec72-4d63-8281-a7089f1230ba kind: Event lastTimestamp: "2026-07-02T18:50:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e22cf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:50:16Z" name: error-404-isvc-e22cf.18be8d9f18974123 namespace: kserve-ci-e2e-test resourceVersion: "19799" uid: a2a90ad1-3427-4df3-97fb-f586520ccfcc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e22cf namespace: kserve-ci-e2e-test resourceVersion: "20407" uid: 365ff1d9-ec72-4d63-8281-a7089f1230ba kind: Event lastTimestamp: "2026-07-02T18:51:05Z" message: InferenceService [error-404-isvc-e22cf] is Ready metadata: creationTimestamp: "2026-07-02T18:51:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:05Z" name: error-404-isvc-e22cf.18be8daa7c51fd10 namespace: kserve-ci-e2e-test resourceVersion: "20408" uid: 1ca54c44-ad5d-484a-92b8-27f6f23b6545 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:41:41.920955Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: fefe989b-4300-4eb7-bb66-816c42550acc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e8b76-predictor-756c9998f6-2lnkl to ip-10-0-140-68.ec2.internal metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:41:41Z" name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl.18be8d272f3b7ddb namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 07253a6f-82a4-48fb-b9d6-0f2cce0d7125 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: fefe989b-4300-4eb7-bb66-816c42550acc kind: Event lastTimestamp: "2026-07-02T18:41:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e8b76-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:42Z" name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl.18be8d273bce42bd namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: 32876cc9-21f9-489e-ada3-2de78276bb27 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: fefe989b-4300-4eb7-bb66-816c42550acc kind: Event lastTimestamp: "2026-07-02T18:41:42Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:41:42Z" name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl.18be8d276c8e5ef5 namespace: kserve-ci-e2e-test resourceVersion: "16040" uid: 0135dd7e-832c-488a-b05e-b9ea1df8e609 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: fefe989b-4300-4eb7-bb66-816c42550acc kind: Event lastTimestamp: "2026-07-02T18:41:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:42Z" name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl.18be8d276dca868a namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: 287d16eb-990d-484f-b638-10af8d0eda06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: fefe989b-4300-4eb7-bb66-816c42550acc kind: Event lastTimestamp: "2026-07-02T18:41:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:43Z" name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl.18be8d2770a46351 namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 01a7a853-680e-4681-ac6a-7a749cea3a3e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: fefe989b-4300-4eb7-bb66-816c42550acc kind: Event lastTimestamp: "2026-07-02T18:41:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:43Z" name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl.18be8d277115ce37 namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: cbb29aef-6def-4fcf-817f-1e00604c9fa4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: fefe989b-4300-4eb7-bb66-816c42550acc kind: Event lastTimestamp: "2026-07-02T18:41:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:43Z" name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl.18be8d27712175f8 namespace: kserve-ci-e2e-test resourceVersion: "16045" uid: 9ccd60f0-f7f9-4609-b3b2-64f0eefd5968 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: fefe989b-4300-4eb7-bb66-816c42550acc kind: Event lastTimestamp: "2026-07-02T18:41:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:43Z" name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl.18be8d27748b93e5 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: c7dcf250-98a6-42a8-baa4-8cff4ef723e7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: fefe989b-4300-4eb7-bb66-816c42550acc kind: Event lastTimestamp: "2026-07-02T18:41:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:43Z" name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl.18be8d2774fb5745 namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 8eda984c-ab14-4f69-8b11-bbd6fbc2c281 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:41:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: fefe989b-4300-4eb7-bb66-816c42550acc kind: Event lastTimestamp: "2026-07-02T18:42:20Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:42:20Z" name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl.18be8d27d6d1ca79 namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: 8ad6999b-7ebc-4ed2-95f5-af70143c1d45 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: fefe989b-4300-4eb7-bb66-816c42550acc kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:56Z" name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl.18be8da864478d7e namespace: kserve-ci-e2e-test resourceVersion: "20270" uid: 481ce65c-6c2f-4679-8e98-0c444e6b1bd0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: fefe989b-4300-4eb7-bb66-816c42550acc kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:56Z" name: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl.18be8da8644ac83e namespace: kserve-ci-e2e-test resourceVersion: "20271" uid: 6c52e45e-1f71-4402-8214-d87c9c973759 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e8b76-predictor-756c9998f6 namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 1cf1fc2c-b6de-4d4b-be8f-a86fd99acaf4 kind: Event lastTimestamp: "2026-07-02T18:41:41Z" message: 'Created pod: error-404-isvc-e8b76-predictor-756c9998f6-2lnkl' metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:41Z" name: error-404-isvc-e8b76-predictor-756c9998f6.18be8d272ed644fe namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: 26f961c4-1d24-468f-96ec-33caea200b5b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e8b76-predictor namespace: kserve-ci-e2e-test resourceVersion: "15988" uid: e853a4e9-b655-4b31-bbeb-e0dfae540d1b kind: Event lastTimestamp: "2026-07-02T18:41:41Z" message: Scaled up replica set error-404-isvc-e8b76-predictor-756c9998f6 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:41Z" name: error-404-isvc-e8b76-predictor.18be8d272d38c28e namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 83d843f8-faf5-48cd-bfe2-a4f05c99149c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:41:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e8b76-predictor namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: ec77f7ee-02bd-4116-9a62-c6f9658f2a32 kind: Event lastTimestamp: "2026-07-02T18:42:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:11Z" name: error-404-isvc-e8b76-predictor.18be8d2aaba30516 namespace: kserve-ci-e2e-test resourceVersion: "16342" uid: 3bca6631-410f-42d1-93a2-98f7b71e2ce5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:41:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e8b76-predictor namespace: kserve-ci-e2e-test resourceVersion: "15990" uid: ec77f7ee-02bd-4116-9a62-c6f9658f2a32 kind: Event lastTimestamp: "2026-07-02T18:42:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:11Z" name: error-404-isvc-e8b76-predictor.18be8d2aabe1fd26 namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: 371dbf96-1621-4b8b-a793-bdd679068313 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:42:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e8b76-predictor namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: ec77f7ee-02bd-4116-9a62-c6f9658f2a32 kind: Event lastTimestamp: "2026-07-02T18:42:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:56Z" name: error-404-isvc-e8b76-predictor.18be8d31a92fb2df namespace: kserve-ci-e2e-test resourceVersion: "16728" uid: f36f3b4c-ddb8-49ee-a6c4-24acfb209942 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:42:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e8b76-predictor namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: ec77f7ee-02bd-4116-9a62-c6f9658f2a32 kind: Event lastTimestamp: "2026-07-02T18:42:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:42:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:56Z" name: error-404-isvc-e8b76-predictor.18be8d31a98e0d1c namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 696a603b-e674-4910-b17a-7b7cceb26219 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e8b76 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 9841551d-6b4f-402c-8d6f-f2ff486ff62c kind: Event lastTimestamp: "2026-07-02T18:42:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:42:30Z" name: error-404-isvc-e8b76.18be8d272d388ef0 namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: 81a7ece0-287c-4cba-b567-b83e268133ce reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e8b76 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 9841551d-6b4f-402c-8d6f-f2ff486ff62c kind: Event lastTimestamp: "2026-07-02T18:41:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-e8b76": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e8b76": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:41:41Z" name: error-404-isvc-e8b76.18be8d272dd47cb7 namespace: kserve-ci-e2e-test resourceVersion: "15996" uid: 648e9728-0f68-4cba-af10-905070496627 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e8b76 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 9841551d-6b4f-402c-8d6f-f2ff486ff62c kind: Event lastTimestamp: "2026-07-02T18:41:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e8b76": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:41:41Z" name: error-404-isvc-e8b76.18be8d272dd4bd70 namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 0b5ad4c0-8830-4108-b72e-a8a451464eaf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e8b76 namespace: kserve-ci-e2e-test resourceVersion: "16497" uid: 9841551d-6b4f-402c-8d6f-f2ff486ff62c kind: Event lastTimestamp: "2026-07-02T18:42:30Z" message: InferenceService [error-404-isvc-e8b76] is Ready metadata: creationTimestamp: "2026-07-02T18:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:42:30Z" name: error-404-isvc-e8b76.18be8d3291e0b9a1 namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 40e78ea0-7b29-4b25-83e2-d872792cdc89 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:39:11.786996Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14207" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr to ip-10-0-140-68.ec2.internal metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:39:11Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d043a8d1610 namespace: kserve-ci-e2e-test resourceVersion: "14211" uid: fd6a0b0a-8dfa-4f28-9334-c40b26dd9782 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:39:12Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:39:12Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d04545e53fc namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: 01b596e4-218f-4ae0-b0a9-a3abea96a065 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:39:12Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2c3a0bd923919758d63cc7ba0fc3688cc68cf683298686e2223d6e819688e8b6" already present on machine metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:12Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d0455ad4f9c namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: 32fd8929-b526-4f95-9403-96399af4b41a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:39:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:12Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d04595606c5 namespace: kserve-ci-e2e-test resourceVersion: "14310" uid: 18bbe6bc-5026-453a-b6c5-02cbdb2756b6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:39:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:12Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d045a1d693a namespace: kserve-ci-e2e-test resourceVersion: "14312" uid: cc52871e-070d-4c2e-961f-7042d067d88d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:39:17Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1698" metadata: creationTimestamp: "2026-07-02T18:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:17Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d05aa4a7070 namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: 623d2210-03d6-499b-aec0-397f7e9495da reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:39:35Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1698" in 17.873s (17.873s including waiting). Image size: 1555871138 bytes.' metadata: creationTimestamp: "2026-07-02T18:39:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:35Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d09d39be6aa namespace: kserve-ci-e2e-test resourceVersion: "14554" uid: f4ddd6c4-5a51-46f4-a57c-f28319a4cefd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:39:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:44Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d0bd796e49a namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: 2cfdfaac-d629-4a5c-8a34-9b5b2ca3506b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:39:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:44Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d0bd92d8627 namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: 4eead620-b6c7-4441-919c-b2c738b1f3be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:39:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T18:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:44Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d0bd93aab54 namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 2bcf0b91-aa56-4161-85c8-a60897b52b5e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:39:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.403s (3.403s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T18:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:47Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d0ca4157d71 namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 42099d31-38ce-4e52-a80a-16b6518a0106 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:39:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:48Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d0ca9c580ab namespace: kserve-ci-e2e-test resourceVersion: "14668" uid: 1300e0cb-0c43-4a56-8a25-89cb15621349 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:39:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:48Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d0caafd7888 namespace: kserve-ci-e2e-test resourceVersion: "14671" uid: b4e8a5a9-8723-4027-a181-0d924830db6f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T18:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:41:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:44Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d0cb46d08bb namespace: kserve-ci-e2e-test resourceVersion: "16064" uid: 7b3018e3-2bea-4785-9ebb-b7065d66d598 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:41:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:41Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d272ad25dce namespace: kserve-ci-e2e-test resourceVersion: "15980" uid: 2760ea4c-f613-4064-a569-9d0d6633e60d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:41:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:41Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d272ad31c80 namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: db9306ac-1729-4189-abbf-f281794fd872 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr namespace: kserve-ci-e2e-test resourceVersion: "14209" uid: 43aa4427-6e7b-40ba-9e27-6a4fa1d3fe97 kind: Event lastTimestamp: "2026-07-02T18:41:44Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:44Z" name: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr.18be8d27b6f8bdf6 namespace: kserve-ci-e2e-test resourceVersion: "16060" uid: 8064bef2-84a5-4b0d-a96c-401ee1c306b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-8b777f59f namespace: kserve-ci-e2e-test resourceVersion: "14198" uid: 81a0d5fd-1ca3-4d51-9720-3a80e0391f08 kind: Event lastTimestamp: "2026-07-02T18:39:11Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-8b777f59f-m4sfr' metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:11Z" name: isvc-sklearn-graph-1-predictor-8b777f59f.18be8d043a2a6f89 namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 88265d52-d435-4edd-88e8-e9993525ae5a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14196" uid: 415b6fbe-5b57-4343-874e-3786bbf2dc07 kind: Event lastTimestamp: "2026-07-02T18:39:11Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-8b777f59f from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:11Z" name: isvc-sklearn-graph-1-predictor.18be8d04371c905d namespace: kserve-ci-e2e-test resourceVersion: "14199" uid: 95df8704-1498-437a-a070-d7ba14e4000f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T18:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: f949d400-54e1-494f-b51b-f1c3974c55e5 kind: Event lastTimestamp: "2026-07-02T18:40:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:26Z" name: isvc-sklearn-graph-1-predictor.18be8d07b57dd46d namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: 2792a3f0-ef51-4f70-964e-fb1948a24a7f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T18:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14197" uid: f949d400-54e1-494f-b51b-f1c3974c55e5 kind: Event lastTimestamp: "2026-07-02T18:40:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:26Z" name: isvc-sklearn-graph-1-predictor.18be8d07b5d5029d namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: fc2229cd-1793-4200-83a7-3668a5b8711f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T18:40:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f949d400-54e1-494f-b51b-f1c3974c55e5 kind: Event lastTimestamp: "2026-07-02T18:41:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:26Z" name: isvc-sklearn-graph-1-predictor.18be8d1231b7c718 namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 952a2386-a558-46fc-9e6c-b2f070ba59e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T18:40:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: f949d400-54e1-494f-b51b-f1c3974c55e5 kind: Event lastTimestamp: "2026-07-02T18:41:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:26Z" name: isvc-sklearn-graph-1-predictor.18be8d1232146646 namespace: kserve-ci-e2e-test resourceVersion: "15713" uid: 39841c6c-b5d3-4cd2-91f4-f061c79d0b48 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: d8748da5-4c14-4a0f-b4a7-0bad8a27f369 kind: Event lastTimestamp: "2026-07-02T18:41:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:41:04Z" name: isvc-sklearn-graph-1.18be8d0437297735 namespace: kserve-ci-e2e-test resourceVersion: "15528" uid: 2768e0f2-cd2d-48ab-928f-d51a35a89a5f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15525" uid: d8748da5-4c14-4a0f-b4a7-0bad8a27f369 kind: Event lastTimestamp: "2026-07-02T18:41:04Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-02T18:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:41:04Z" name: isvc-sklearn-graph-1.18be8d1e6ef7bd91 namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 8b6c28fc-f91a-44c7-881c-6ed34f932692 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:39:12.090323Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd to ip-10-0-140-68.ec2.internal metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:39:12Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d044ca1782a namespace: kserve-ci-e2e-test resourceVersion: "14291" uid: fc0f371f-c6ac-4fdb-b8d7-6340969e1b87 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:39:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:12Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d0458427168 namespace: kserve-ci-e2e-test resourceVersion: "14308" uid: 41509d3a-b598-483f-b5cf-bf30ba079ab7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:39:13Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:39:13Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d048f20710e namespace: kserve-ci-e2e-test resourceVersion: "14342" uid: 96f6b18c-d2a7-4c66-996d-6a1623fa5a36 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:39:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2c3a0bd923919758d63cc7ba0fc3688cc68cf683298686e2223d6e819688e8b6" already present on machine metadata: creationTimestamp: "2026-07-02T18:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:13Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d0490c46cdb namespace: kserve-ci-e2e-test resourceVersion: "14346" uid: 111ba6a4-a57c-4014-be7b-9a8abd51cbb6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:39:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T18:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:13Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d0495c4c9a6 namespace: kserve-ci-e2e-test resourceVersion: "14347" uid: ccedf73f-f9fb-49a1-9af6-e7fde8252894 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:39:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T18:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:13Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d04969da3e6 namespace: kserve-ci-e2e-test resourceVersion: "14348" uid: 3575a3c1-0858-4736-8cf7-6e7ceb92e8d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:39:17Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1698" metadata: creationTimestamp: "2026-07-02T18:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:17Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d05aa64a839 namespace: kserve-ci-e2e-test resourceVersion: "14394" uid: 2c857140-888b-40ec-a291-e1a6b200aa3d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:39:44Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1698" in 26.435s (26.435s including waiting). Image size: 1555871138 bytes.' metadata: creationTimestamp: "2026-07-02T18:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:44Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d0bd20d23e3 namespace: kserve-ci-e2e-test resourceVersion: "14622" uid: f8b0a36c-6326-47fb-b6fa-378ac5bdcee4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:39:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:44Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d0bdb033700 namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: 1ef6a495-6cc3-478d-81ee-773fb9ba704c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:39:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:44Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d0bdc358388 namespace: kserve-ci-e2e-test resourceVersion: "14631" uid: 53435f9c-c7fa-468a-905b-9f4368914349 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:39:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T18:39:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:44Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d0bdc41e56b namespace: kserve-ci-e2e-test resourceVersion: "14632" uid: 362cc137-24d8-4168-9a32-a56517400d04 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:39:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.373s (3.373s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T18:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:47Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d0ca55cde42 namespace: kserve-ci-e2e-test resourceVersion: "14661" uid: 485b23a5-5a8d-4675-80e0-0af0aac1624d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:39:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:48Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d0ca9ab2502 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: 13380c46-28fe-4bd6-aebd-c9e867ff3a46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:39:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:48Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d0caa7ce3cf namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: facd609b-bea7-4fd6-984d-d0263d8cc416 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T18:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:41:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:44Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d0cb4a31aa1 namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 0fe9ecf0-deb9-44ab-b773-6103f76b42c4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:41:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:41Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d2727b79a57 namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 9b8f2df7-3218-482e-817b-461ad1390b83 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:41:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:41Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d2727b82c91 namespace: kserve-ci-e2e-test resourceVersion: "15954" uid: 4a0ce7a0-cb98-4ec3-b63a-6e5c7cac267a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 1b12d2ba-3425-4dc3-b91d-e709f8436b48 kind: Event lastTimestamp: "2026-07-02T18:41:44Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:44Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd.18be8d27b6f9d606 namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: e1686add-b69b-49be-a7f3-fa88def3ce08 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-6c85f75dc7 namespace: kserve-ci-e2e-test resourceVersion: "14277" uid: 0a5f0032-8847-4a7d-a069-a85b338a3bbe kind: Event lastTimestamp: "2026-07-02T18:39:12Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-6c85f75dc7-mlggd' metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:12Z" name: isvc-sklearn-graph-2-predictor-6c85f75dc7.18be8d044c39eef3 namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: 51b9f4d0-6c5d-4027-b9f1-9ea3fa79f1a1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14274" uid: 1f58a38f-ceb8-4806-9ca5-0ad37ebea9eb kind: Event lastTimestamp: "2026-07-02T18:39:12Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-6c85f75dc7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:12Z" name: isvc-sklearn-graph-2-predictor.18be8d0449971834 namespace: kserve-ci-e2e-test resourceVersion: "14278" uid: 5774b78b-405a-43c6-8156-2dad399ea899 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T18:39:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: cbe86d95-6178-4de4-823a-b98396f46ac6 kind: Event lastTimestamp: "2026-07-02T18:40:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:27Z" name: isvc-sklearn-graph-2-predictor.18be8d07c80adcd2 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: aef7d8cc-55f3-4026-9fa6-82ed8cd3fd70 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T18:39:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14276" uid: cbe86d95-6178-4de4-823a-b98396f46ac6 kind: Event lastTimestamp: "2026-07-02T18:40:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:39:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:27Z" name: isvc-sklearn-graph-2-predictor.18be8d07c86aa050 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 62eba65b-d9c8-4bb7-bdba-9d1a3b7cb9aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T18:40:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: cbe86d95-6178-4de4-823a-b98396f46ac6 kind: Event lastTimestamp: "2026-07-02T18:41:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:27Z" name: isvc-sklearn-graph-2-predictor.18be8d1941d23931 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 31ec2c19-6311-46c1-a118-9ee889803d66 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T18:40:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14482" uid: cbe86d95-6178-4de4-823a-b98396f46ac6 kind: Event lastTimestamp: "2026-07-02T18:41:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:27Z" name: isvc-sklearn-graph-2-predictor.18be8d194239c5ba namespace: kserve-ci-e2e-test resourceVersion: "15718" uid: d97522cd-ebe8-405d-90b2-60666502772e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14256" uid: 120e0a35-d45d-45f1-98d2-586bc05b4046 kind: Event lastTimestamp: "2026-07-02T18:41:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:41:04Z" name: isvc-sklearn-graph-2.18be8d04499e25a8 namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: a70a94a1-16d7-4f2c-abd7-0e8edde74c8c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15521" uid: 120e0a35-d45d-45f1-98d2-586bc05b4046 kind: Event lastTimestamp: "2026-07-02T18:41:04Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-02T18:41:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:41:04Z" name: isvc-sklearn-graph-2.18be8d1e6c64e61d namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: 0c6f9501-af7a-4b7f-8816-8e84502687f5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:39:11.991484Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-c5zrq to ip-10-0-140-68.ec2.internal metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:39:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d0446bd5a2a namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: 862c4066-87d2-4874-80d3-ce435a0b125d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14275" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:39:12Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:39:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d04608fbdee namespace: kserve-ci-e2e-test resourceVersion: "14322" uid: bb3274cc-e51a-4edf-849d-ac0d1e01efdf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:39:12Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:2c3a0bd923919758d63cc7ba0fc3688cc68cf683298686e2223d6e819688e8b6" already present on machine metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d0461bd7079 namespace: kserve-ci-e2e-test resourceVersion: "14324" uid: 73f762ba-9e11-44a3-8acf-88879b54118e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:39:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d04648e453e namespace: kserve-ci-e2e-test resourceVersion: "14326" uid: 93d167db-4ee2-4605-b760-2abbc59c4ea9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:39:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T18:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d0465080b85 namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: 535a9c47-1a59-4282-a4cf-7632463aa2e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:39:17Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-02T18:39:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:17Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d05aa2df09d namespace: kserve-ci-e2e-test resourceVersion: "14391" uid: 3da934da-36c3-480a-be80-4884cb753205 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:39:45Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 27.06s (27.06s including waiting). Image size: 1305293048 bytes.' metadata: creationTimestamp: "2026-07-02T18:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d0bf71f611e namespace: kserve-ci-e2e-test resourceVersion: "14635" uid: e0150b96-2792-4bcc-8107-2350899afe45 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:39:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d0bfcf5204a namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: 13ce7a44-3117-4ca7-aca4-98677b391b73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:39:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d0bfea378d6 namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: 4ec15034-31f3-43e7-82d7-2d6174048ba7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:39:45Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T18:39:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d0bfeb47310 namespace: kserve-ci-e2e-test resourceVersion: "14639" uid: a54ae5d8-406f-4e9a-af07-2135804a37fe reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:39:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.809s (2.809s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T18:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:47Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d0ca62acc2a namespace: kserve-ci-e2e-test resourceVersion: "14662" uid: 33ad182a-8e6f-4c0b-8952-a65ee5e9e2d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:39:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d0ca9b909e4 namespace: kserve-ci-e2e-test resourceVersion: "14667" uid: dfac15d7-63c4-4591-8abe-2bf908b89323 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:39:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:48Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d0caa7b7609 namespace: kserve-ci-e2e-test resourceVersion: "14669" uid: e9a8ae5d-3932-45e3-8a16-b37ed2506df6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T18:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:41:45Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d0cf0885ef8 namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: e2cca7d8-2158-4bf4-94fb-0f001e04ba60 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:41:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d2724ad76fb namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: f2e223ae-579a-4012-9e5d-6b4a0f558354 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:41:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d2724aefabd namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: d0bf5767-cf41-43a4-99b3-35801ae49c9f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: a245f3bd-e1b2-4adb-a2d2-f38e34b31858 kind: Event lastTimestamp: "2026-07-02T18:41:45Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-c5zrq.18be8d27f2d4149a namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: ddd355a2-80f3-4327-bb8c-4123cf8ca20c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-68.ec2.internal source: component: kubelet host: ip-10-0-140-68.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14246" uid: 425b644f-b6e4-452c-99ee-e6ae5a6eecfb kind: Event lastTimestamp: "2026-07-02T18:39:11Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-c5zrq' metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:11Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be8d044651e2f3 namespace: kserve-ci-e2e-test resourceVersion: "14259" uid: 6d2d9a37-8d42-49a5-9618-cad14d1f5320 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14244" uid: fa70a5c1-a94a-4aaf-9d4b-a5260d833244 kind: Event lastTimestamp: "2026-07-02T18:39:11Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:11Z" name: isvc-xgboost-graph-predictor.18be8d0443368ead namespace: kserve-ci-e2e-test resourceVersion: "14247" uid: 2ba40d66-9477-4752-b2a4-5ae5697387d9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: fe9db448-4396-4cae-ace5-d7cfa20e3325 kind: Event lastTimestamp: "2026-07-02T18:39:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:56Z" name: isvc-xgboost-graph-predictor.18be8d07c1d1b63b namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: b91bba3c-45da-424f-a4bd-f82e97cdbd44 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14245" uid: fe9db448-4396-4cae-ace5-d7cfa20e3325 kind: Event lastTimestamp: "2026-07-02T18:39:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:56Z" name: isvc-xgboost-graph-predictor.18be8d07c2304728 namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: 33a8121e-9c98-4d2a-b134-8ec918e492c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:40:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: fe9db448-4396-4cae-ace5-d7cfa20e3325 kind: Event lastTimestamp: "2026-07-02T18:41:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:27Z" name: isvc-xgboost-graph-predictor.18be8d123df7818c namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: bde471c6-f715-441f-9ded-45410062c9b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:40:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: fe9db448-4396-4cae-ace5-d7cfa20e3325 kind: Event lastTimestamp: "2026-07-02T18:41:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:27Z" name: isvc-xgboost-graph-predictor.18be8d123e5327e5 namespace: kserve-ci-e2e-test resourceVersion: "15716" uid: 9c369d90-1518-47be-84f1-61b24acb557e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14229" uid: 20eef21b-c429-4ccc-bf7c-94b8457d90d8 kind: Event lastTimestamp: "2026-07-02T18:40:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:40:55Z" name: isvc-xgboost-graph.18be8d04434302a5 namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: 028bfc7d-0921-4cd0-b96d-ea485df76572 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: 20eef21b-c429-4ccc-bf7c-94b8457d90d8 kind: Event lastTimestamp: "2026-07-02T18:40:55Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-02T18:40:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:40:55Z" name: isvc-xgboost-graph.18be8d1c53f002e3 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: a79cdd95-5aa5-442a-aed6-9a14d7bdef0f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:41:31.551743Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5bd77c7cd9-6j798 namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: 73797b34-dc9c-4c45-9ecc-959e27908789 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-5bd77c7cd9-6j798 to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:41:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:41:31Z" name: model-chainer-5bd77c7cd9-6j798.18be8d24c52ddbbe namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: 89327180-8c1c-4fc3-a640-dd321d6644d2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:31Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5bd77c7cd9-6j798 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 73797b34-dc9c-4c45-9ecc-959e27908789 kind: Event lastTimestamp: "2026-07-02T18:41:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T18:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:31Z" name: model-chainer-5bd77c7cd9-6j798.18be8d24d09d9bbf namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: 12e1d710-4758-48dc-9f9f-bf57dbbbb5e1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:32Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5bd77c7cd9-6j798 namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: 73797b34-dc9c-4c45-9ecc-959e27908789 kind: Event lastTimestamp: "2026-07-02T18:41:32Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:41:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:41:32Z" name: model-chainer-5bd77c7cd9-6j798.18be8d2502cf4ea7 namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: edf44e95-817d-48db-b977-58c245f6cde0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5bd77c7cd9-6j798 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 73797b34-dc9c-4c45-9ecc-959e27908789 kind: Event lastTimestamp: "2026-07-02T18:41:32Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:75c3c56106a5b8ed3fd4cb951cdacfbe184ab3e990f6497776c0954efdfa10b2" already present on machine metadata: creationTimestamp: "2026-07-02T18:41:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:32Z" name: model-chainer-5bd77c7cd9-6j798.18be8d2504095cc0 namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: 0a4966c1-ee7d-4cf7-a205-5a1d1bf38ff2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5bd77c7cd9-6j798 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 73797b34-dc9c-4c45-9ecc-959e27908789 kind: Event lastTimestamp: "2026-07-02T18:41:32Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-02T18:41:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:32Z" name: model-chainer-5bd77c7cd9-6j798.18be8d2507493e8e namespace: kserve-ci-e2e-test resourceVersion: "15789" uid: a8469b3c-7f9a-4d97-82f5-09d86a1490e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5bd77c7cd9-6j798 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 73797b34-dc9c-4c45-9ecc-959e27908789 kind: Event lastTimestamp: "2026-07-02T18:41:32Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-02T18:41:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:32Z" name: model-chainer-5bd77c7cd9-6j798.18be8d2507bb9b6d namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: cd6401cf-24e2-4b87-9f91-10b30db5afc8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5bd77c7cd9-6j798 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 73797b34-dc9c-4c45-9ecc-959e27908789 kind: Event lastTimestamp: "2026-07-02T18:41:41Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:41Z" name: model-chainer-5bd77c7cd9-6j798.18be8d271f8f9f6d namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: 9e5c0ae8-61f4-4985-97fd-9801aa0d851c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:41:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5bd77c7cd9-6j798 namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 73797b34-dc9c-4c45-9ecc-959e27908789 kind: Event lastTimestamp: "2026-07-02T18:42:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T18:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:42:09Z" name: model-chainer-5bd77c7cd9-6j798.18be8d27c18474fb namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 0bc8809d-eef1-4b8f-85eb-0a8b39df99dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-5bd77c7cd9 namespace: kserve-ci-e2e-test resourceVersion: "15757" uid: 68b60089-8297-4cd3-9ec0-0b0dfed58531 kind: Event lastTimestamp: "2026-07-02T18:41:31Z" message: 'Created pod: model-chainer-5bd77c7cd9-6j798' metadata: creationTimestamp: "2026-07-02T18:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:31Z" name: model-chainer-5bd77c7cd9.18be8d24c4c41381 namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 3fca9e2e-a4fb-40de-9122-c22aa41c75ee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15755" uid: 79a2ddbb-d9d8-4ad4-9e1a-219a6eadb287 kind: Event lastTimestamp: "2026-07-02T18:41:31Z" message: Scaled up replica set model-chainer-5bd77c7cd9 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:31Z" name: model-chainer.18be8d24c3fe5395 namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: c02e580a-e97b-49bb-9d1d-f34854b71d5f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: c534fff9-c5a5-4aa8-b2dd-51deadbe798f kind: Event lastTimestamp: "2026-07-02T18:41:31Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-02T18:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:41:31Z" name: model-chainer.18be8d24c8b5299c namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: e02f9d07-9ff8-4de3-bc08-0137cdb83942 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:51:56.750702Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-94e8c-5775877b49-zkm54 namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: d3d06625-9504-4963-a5c6-656c3fc8fb94 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-94e8c-5775877b49-zkm54 to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:51:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:51:56Z" name: sequence-graph-94e8c-5775877b49-zkm54.18be8db655f0d9bf namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: 5cf817fd-870f-4a6b-a08e-1ca42f35ed9d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:57Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-94e8c-5775877b49-zkm54 namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: d3d06625-9504-4963-a5c6-656c3fc8fb94 kind: Event lastTimestamp: "2026-07-02T18:51:57Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:51:57Z" name: sequence-graph-94e8c-5775877b49-zkm54.18be8db670279b57 namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 754a5136-f1a7-43a3-8804-01512a19184e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-94e8c} kind: Pod name: sequence-graph-94e8c-5775877b49-zkm54 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: d3d06625-9504-4963-a5c6-656c3fc8fb94 kind: Event lastTimestamp: "2026-07-02T18:51:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:75c3c56106a5b8ed3fd4cb951cdacfbe184ab3e990f6497776c0954efdfa10b2" already present on machine metadata: creationTimestamp: "2026-07-02T18:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:57Z" name: sequence-graph-94e8c-5775877b49-zkm54.18be8db6716faefd namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: 92d5aa4a-7c8b-43f4-88e7-8ec1ffea9be3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-94e8c} kind: Pod name: sequence-graph-94e8c-5775877b49-zkm54 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: d3d06625-9504-4963-a5c6-656c3fc8fb94 kind: Event lastTimestamp: "2026-07-02T18:51:57Z" message: 'Created container: sequence-graph-94e8c' metadata: creationTimestamp: "2026-07-02T18:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:57Z" name: sequence-graph-94e8c-5775877b49-zkm54.18be8db673e142cb namespace: kserve-ci-e2e-test resourceVersion: "21062" uid: bdf8881e-8cb8-4454-9b08-eda129140e9d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-94e8c} kind: Pod name: sequence-graph-94e8c-5775877b49-zkm54 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: d3d06625-9504-4963-a5c6-656c3fc8fb94 kind: Event lastTimestamp: "2026-07-02T18:51:57Z" message: Started container sequence-graph-94e8c metadata: creationTimestamp: "2026-07-02T18:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:57Z" name: sequence-graph-94e8c-5775877b49-zkm54.18be8db6744ffbe8 namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: c394a623-120a-4a3a-8488-5a45650a2c13 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-94e8c} kind: Pod name: sequence-graph-94e8c-5775877b49-zkm54 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: d3d06625-9504-4963-a5c6-656c3fc8fb94 kind: Event lastTimestamp: "2026-07-02T18:52:06Z" message: Stopping container sequence-graph-94e8c metadata: creationTimestamp: "2026-07-02T18:52:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:06Z" name: sequence-graph-94e8c-5775877b49-zkm54.18be8db8af88bb2c namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: 7c4f5dbe-fa0f-443a-8a78-70b3b50ba759 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:52:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-94e8c} kind: Pod name: sequence-graph-94e8c-5775877b49-zkm54 namespace: kserve-ci-e2e-test resourceVersion: "21032" uid: d3d06625-9504-4963-a5c6-656c3fc8fb94 kind: Event lastTimestamp: "2026-07-02T18:52:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T18:52:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:33Z" name: sequence-graph-94e8c-5775877b49-zkm54.18be8db90799ec1c namespace: kserve-ci-e2e-test resourceVersion: "21547" uid: cadd309b-b05e-4341-8ebc-093e627c02dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-94e8c-5775877b49 namespace: kserve-ci-e2e-test resourceVersion: "21026" uid: 407776a9-2b18-4a09-b34c-e73a8d1b7f08 kind: Event lastTimestamp: "2026-07-02T18:51:56Z" message: 'Created pod: sequence-graph-94e8c-5775877b49-zkm54' metadata: creationTimestamp: "2026-07-02T18:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:56Z" name: sequence-graph-94e8c-5775877b49.18be8db655548cdb namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 8163cc94-6038-48ee-a056-3c8c2324a440 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-94e8c namespace: kserve-ci-e2e-test resourceVersion: "21024" uid: 835b3b8b-29f1-4d75-892b-89f74828423a kind: Event lastTimestamp: "2026-07-02T18:51:56Z" message: Scaled up replica set sequence-graph-94e8c-5775877b49 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:56Z" name: sequence-graph-94e8c.18be8db654b4926d namespace: kserve-ci-e2e-test resourceVersion: "21027" uid: 76941cdf-aaa0-4631-9af0-8c7ec8144f8c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-94e8c namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: d7764118-1381-4b2c-a0f7-0e7882616157 kind: Event lastTimestamp: "2026-07-02T18:51:56Z" message: InferenceGraph [sequence-graph-94e8c] is Ready metadata: creationTimestamp: "2026-07-02T18:51:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:56Z" name: sequence-graph-94e8c.18be8db65a057350 namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: 28e4bd7d-c00f-4cc4-bb1f-143fd3592b15 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:53:07.058964Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ae135-78dffc7b7d-86kjs namespace: kserve-ci-e2e-test resourceVersion: "21837" uid: ef9f4197-c9d1-48b1-b605-52b58b3e8e59 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-ae135-78dffc7b7d-86kjs to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:53:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:53:07Z" name: sequence-graph-ae135-78dffc7b7d-86kjs.18be8dc6b4a3d4f7 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 21acb367-0ea2-465c-85a4-c82892f9be66 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:53:07Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ae135-78dffc7b7d-86kjs namespace: kserve-ci-e2e-test resourceVersion: "21847" uid: ef9f4197-c9d1-48b1-b605-52b58b3e8e59 kind: Event lastTimestamp: "2026-07-02T18:53:07Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:53:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:53:07Z" name: sequence-graph-ae135-78dffc7b7d-86kjs.18be8dc6ce47204f namespace: kserve-ci-e2e-test resourceVersion: "21857" uid: df0f254f-0bbc-4168-bbea-c87afd6af39f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:53:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ae135} kind: Pod name: sequence-graph-ae135-78dffc7b7d-86kjs namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: ef9f4197-c9d1-48b1-b605-52b58b3e8e59 kind: Event lastTimestamp: "2026-07-02T18:53:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:75c3c56106a5b8ed3fd4cb951cdacfbe184ab3e990f6497776c0954efdfa10b2" already present on machine metadata: creationTimestamp: "2026-07-02T18:53:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:53:07Z" name: sequence-graph-ae135-78dffc7b7d-86kjs.18be8dc6cf778698 namespace: kserve-ci-e2e-test resourceVersion: "21859" uid: d1acedf2-441c-496e-824b-47a1b4c1eaad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:53:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ae135} kind: Pod name: sequence-graph-ae135-78dffc7b7d-86kjs namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: ef9f4197-c9d1-48b1-b605-52b58b3e8e59 kind: Event lastTimestamp: "2026-07-02T18:53:07Z" message: 'Created container: sequence-graph-ae135' metadata: creationTimestamp: "2026-07-02T18:53:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:53:07Z" name: sequence-graph-ae135-78dffc7b7d-86kjs.18be8dc6d27b1b26 namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: b8abc139-70e0-4df1-a27b-84d4afb92a2f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:53:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ae135} kind: Pod name: sequence-graph-ae135-78dffc7b7d-86kjs namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: ef9f4197-c9d1-48b1-b605-52b58b3e8e59 kind: Event lastTimestamp: "2026-07-02T18:53:07Z" message: Started container sequence-graph-ae135 metadata: creationTimestamp: "2026-07-02T18:53:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:53:07Z" name: sequence-graph-ae135-78dffc7b7d-86kjs.18be8dc6d2e8b25f namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: 4de1d70a-53ba-4161-b98a-180f75dd1323 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ae135} kind: Pod name: sequence-graph-ae135-78dffc7b7d-86kjs namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: ef9f4197-c9d1-48b1-b605-52b58b3e8e59 kind: Event lastTimestamp: "2026-07-02T19:01:21Z" message: Stopping container sequence-graph-ae135 metadata: creationTimestamp: "2026-07-02T19:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:21Z" name: sequence-graph-ae135-78dffc7b7d-86kjs.18be8e39e7418266 namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: bb7d56cf-d606-4903-bcb1-b07a87358d29 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T19:01:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ae135} kind: Pod name: sequence-graph-ae135-78dffc7b7d-86kjs namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: ef9f4197-c9d1-48b1-b605-52b58b3e8e59 kind: Event lastTimestamp: "2026-07-02T19:01:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T19:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:48Z" name: sequence-graph-ae135-78dffc7b7d-86kjs.18be8e3a54221276 namespace: kserve-ci-e2e-test resourceVersion: "26116" uid: 5eb0c2c2-0160-4acf-86f4-f085f44322ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:53:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-ae135-78dffc7b7d namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 11facff7-43c9-4999-8b7b-f5a699b41963 kind: Event lastTimestamp: "2026-07-02T18:53:07Z" message: 'Created pod: sequence-graph-ae135-78dffc7b7d-86kjs' metadata: creationTimestamp: "2026-07-02T18:53:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:53:07Z" name: sequence-graph-ae135-78dffc7b7d.18be8dc6b41936b9 namespace: kserve-ci-e2e-test resourceVersion: "21839" uid: 34a2b49f-1bac-4304-a3e5-ee3bbd8cd7b3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:53:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-ae135 namespace: kserve-ci-e2e-test resourceVersion: "21832" uid: 5dcdc008-70cc-4bb6-9f15-922afbab1cc4 kind: Event lastTimestamp: "2026-07-02T18:53:07Z" message: Scaled up replica set sequence-graph-ae135-78dffc7b7d from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:53:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:53:07Z" name: sequence-graph-ae135.18be8dc6b333b1f7 namespace: kserve-ci-e2e-test resourceVersion: "21835" uid: ae353085-2b7b-4312-b81f-9e007a9dff6b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:53:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-ae135 namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: d951611e-121f-4d09-8c78-5fda7d1b1f83 kind: Event lastTimestamp: "2026-07-02T18:53:07Z" message: InferenceGraph [sequence-graph-ae135] is Ready metadata: creationTimestamp: "2026-07-02T18:53:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:53:07Z" name: sequence-graph-ae135.18be8dc6b6c04ebf namespace: kserve-ci-e2e-test resourceVersion: "21853" uid: ef511512-fbc0-4bdc-9787-c4208e82cf13 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:53:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ae135 namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: 4c6b71ea-c04c-44ce-86ef-fa45d4725762 kind: Event lastTimestamp: "2026-07-02T18:53:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:53:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:53:37Z" name: sequence-graph-ae135.18be8dca318fc542 namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: 4604d8bb-c5f1-44ad-847c-148470a5fd74 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:53:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ae135 namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: 4c6b71ea-c04c-44ce-86ef-fa45d4725762 kind: Event lastTimestamp: "2026-07-02T18:53:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:53:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:53:37Z" name: sequence-graph-ae135.18be8dca31f202a2 namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 9000218a-35af-40e4-b639-48456b09ad16 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:42:41.843459Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e8b76-cc658d568-n4p7n namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: 7715988c-7201-4301-9699-e351641dafbf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-e8b76-cc658d568-n4p7n to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:42:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:42:41Z" name: sequence-graph-e8b76-cc658d568-n4p7n.18be8d3522e46441 namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: d1b56ecb-eb27-4377-97e6-968fedb65af6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:42Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e8b76-cc658d568-n4p7n namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 7715988c-7201-4301-9699-e351641dafbf kind: Event lastTimestamp: "2026-07-02T18:42:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-e8b76-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T18:42:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:42:42Z" name: sequence-graph-e8b76-cc658d568-n4p7n.18be8d352fe70db0 namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: 0e1fd375-1c51-41c9-bd3d-382dd52bd4ea reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:42Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e8b76-cc658d568-n4p7n namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: 7715988c-7201-4301-9699-e351641dafbf kind: Event lastTimestamp: "2026-07-02T18:42:42Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:42:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:42:42Z" name: sequence-graph-e8b76-cc658d568-n4p7n.18be8d3560a4d90b namespace: kserve-ci-e2e-test resourceVersion: "16615" uid: a761770d-fae4-4d49-90f1-f15837bdff45 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e8b76} kind: Pod name: sequence-graph-e8b76-cc658d568-n4p7n namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 7715988c-7201-4301-9699-e351641dafbf kind: Event lastTimestamp: "2026-07-02T18:42:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:75c3c56106a5b8ed3fd4cb951cdacfbe184ab3e990f6497776c0954efdfa10b2" already present on machine metadata: creationTimestamp: "2026-07-02T18:42:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:42:42Z" name: sequence-graph-e8b76-cc658d568-n4p7n.18be8d3562009e5c namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 3662e201-69d4-4fd8-a0f3-944974c204ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e8b76} kind: Pod name: sequence-graph-e8b76-cc658d568-n4p7n namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 7715988c-7201-4301-9699-e351641dafbf kind: Event lastTimestamp: "2026-07-02T18:42:42Z" message: 'Created container: sequence-graph-e8b76' metadata: creationTimestamp: "2026-07-02T18:42:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:42:42Z" name: sequence-graph-e8b76-cc658d568-n4p7n.18be8d35657a05b6 namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: 968c6572-34de-4f09-8f90-42e9c0390830 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e8b76} kind: Pod name: sequence-graph-e8b76-cc658d568-n4p7n namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 7715988c-7201-4301-9699-e351641dafbf kind: Event lastTimestamp: "2026-07-02T18:42:42Z" message: Started container sequence-graph-e8b76 metadata: creationTimestamp: "2026-07-02T18:42:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:42:42Z" name: sequence-graph-e8b76-cc658d568-n4p7n.18be8d3565ed90d9 namespace: kserve-ci-e2e-test resourceVersion: "16619" uid: 1c9b3504-8002-48db-aaf9-8a09729fd9ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e8b76} kind: Pod name: sequence-graph-e8b76-cc658d568-n4p7n namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 7715988c-7201-4301-9699-e351641dafbf kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: Stopping container sequence-graph-e8b76 metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:56Z" name: sequence-graph-e8b76-cc658d568-n4p7n.18be8da853c13847 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: d0222400-b70e-44e8-9395-0442cda328d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:50:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e8b76} kind: Pod name: sequence-graph-e8b76-cc658d568-n4p7n namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 7715988c-7201-4301-9699-e351641dafbf kind: Event lastTimestamp: "2026-07-02T18:51:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T18:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:24Z" name: sequence-graph-e8b76-cc658d568-n4p7n.18be8da90dadcebc namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: f562dbc7-d338-4f66-b2a3-0946217f2ada reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-e8b76-cc658d568 namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: 76da8896-4540-4f5e-aacb-3da354b710cb kind: Event lastTimestamp: "2026-07-02T18:42:41Z" message: 'Created pod: sequence-graph-e8b76-cc658d568-n4p7n' metadata: creationTimestamp: "2026-07-02T18:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:41Z" name: sequence-graph-e8b76-cc658d568.18be8d35225a4c28 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: 6bace1da-6f63-496a-bdf0-8f404b82424e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-e8b76 namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: 37b323e7-63a4-4622-8760-b40a682be0de kind: Event lastTimestamp: "2026-07-02T18:42:41Z" message: Scaled up replica set sequence-graph-e8b76-cc658d568 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:41Z" name: sequence-graph-e8b76.18be8d3521948ff6 namespace: kserve-ci-e2e-test resourceVersion: "16584" uid: 1235900d-953c-46b0-8ed6-6677d71369b1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e8b76 namespace: kserve-ci-e2e-test resourceVersion: "16600" uid: e90b4281-6e4d-4ab3-ab00-25b5d7a7b061 kind: Event lastTimestamp: "2026-07-02T18:42:41Z" message: InferenceGraph [sequence-graph-e8b76] is Ready metadata: creationTimestamp: "2026-07-02T18:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:42:41Z" name: sequence-graph-e8b76.18be8d3526daa184 namespace: kserve-ci-e2e-test resourceVersion: "16602" uid: ab33eea4-788f-4f9a-82f8-719a76d9b1b3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e8b76 namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: 54d7f78a-f194-43d3-920a-7b5dc712798e kind: Event lastTimestamp: "2026-07-02T18:42:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:42:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:56Z" name: sequence-graph-e8b76.18be8d38a056bc84 namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: a2bd3f42-0bef-4625-bdce-5b09fc58d312 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e8b76 namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: 54d7f78a-f194-43d3-920a-7b5dc712798e kind: Event lastTimestamp: "2026-07-02T18:42:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:42:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:56Z" name: sequence-graph-e8b76.18be8d38a0b21fba namespace: kserve-ci-e2e-test resourceVersion: "16725" uid: 8df3546d-3c7f-489c-ae73-475016b290ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:43:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e8b76 namespace: kserve-ci-e2e-test resourceVersion: "16724" uid: 54d7f78a-f194-43d3-920a-7b5dc712798e kind: Event lastTimestamp: "2026-07-02T18:43:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:43:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:43:11Z" name: sequence-graph-e8b76.18be8d3c1f4b2522 namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: b618a541-cb4a-4737-9393-d6690df37e55 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:43:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-e8b76 namespace: kserve-ci-e2e-test resourceVersion: "16724" uid: 54d7f78a-f194-43d3-920a-7b5dc712798e kind: Event lastTimestamp: "2026-07-02T18:43:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:43:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:43:11Z" name: sequence-graph-e8b76.18be8d3c1f90a199 namespace: kserve-ci-e2e-test resourceVersion: "16831" uid: ee1bc1a8-7fda-4848-ba9b-9dbdd9c343e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T19:02:52.278024Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c41f9-5bc8d4797c-wprsh namespace: kserve-ci-e2e-test resourceVersion: "26859" uid: 07d94ec8-ec00-4692-a1e4-03fdb47f5ca8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-c41f9-5bc8d4797c-wprsh to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T19:02:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T19:02:52Z" name: splitter-graph-c41f9-5bc8d4797c-wprsh.18be8e4ef66a0217 namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: f68e7d16-0b9a-4e20-9a8e-a2f097743f50 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:52Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c41f9-5bc8d4797c-wprsh namespace: kserve-ci-e2e-test resourceVersion: "26868" uid: 07d94ec8-ec00-4692-a1e4-03fdb47f5ca8 kind: Event lastTimestamp: "2026-07-02T19:02:52Z" message: Add eth0 [10.134.0.53/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T19:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T19:02:52Z" name: splitter-graph-c41f9-5bc8d4797c-wprsh.18be8e4f0fdbad19 namespace: kserve-ci-e2e-test resourceVersion: "26879" uid: 9154f48c-a037-4ec9-9ec2-48a4ec18ce0a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c41f9} kind: Pod name: splitter-graph-c41f9-5bc8d4797c-wprsh namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: 07d94ec8-ec00-4692-a1e4-03fdb47f5ca8 kind: Event lastTimestamp: "2026-07-02T19:02:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:75c3c56106a5b8ed3fd4cb951cdacfbe184ab3e990f6497776c0954efdfa10b2" already present on machine metadata: creationTimestamp: "2026-07-02T19:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:02:52Z" name: splitter-graph-c41f9-5bc8d4797c-wprsh.18be8e4f11170599 namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: a68ef0fb-946c-467c-b834-6f1989eeaf84 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c41f9} kind: Pod name: splitter-graph-c41f9-5bc8d4797c-wprsh namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: 07d94ec8-ec00-4692-a1e4-03fdb47f5ca8 kind: Event lastTimestamp: "2026-07-02T19:02:52Z" message: 'Created container: splitter-graph-c41f9' metadata: creationTimestamp: "2026-07-02T19:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:02:52Z" name: splitter-graph-c41f9-5bc8d4797c-wprsh.18be8e4f147150f5 namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: 52fa8192-fe47-4ff3-953c-69b7351b882a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c41f9} kind: Pod name: splitter-graph-c41f9-5bc8d4797c-wprsh namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: 07d94ec8-ec00-4692-a1e4-03fdb47f5ca8 kind: Event lastTimestamp: "2026-07-02T19:02:52Z" message: Started container splitter-graph-c41f9 metadata: creationTimestamp: "2026-07-02T19:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:02:52Z" name: splitter-graph-c41f9-5bc8d4797c-wprsh.18be8e4f14e00d3a namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: ca14ac0e-ca62-404d-9fcb-d4231219b96d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c41f9} kind: Pod name: splitter-graph-c41f9-5bc8d4797c-wprsh namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: 07d94ec8-ec00-4692-a1e4-03fdb47f5ca8 kind: Event lastTimestamp: "2026-07-02T19:11:06Z" message: Stopping container splitter-graph-c41f9 metadata: creationTimestamp: "2026-07-02T19:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:11:06Z" name: splitter-graph-c41f9-5bc8d4797c-wprsh.18be8ec223276eac namespace: kserve-ci-e2e-test resourceVersion: "30312" uid: 6e7094fd-8093-4fcd-8f67-98ad00629a24 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T19:11:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c41f9} kind: Pod name: splitter-graph-c41f9-5bc8d4797c-wprsh namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: 07d94ec8-ec00-4692-a1e4-03fdb47f5ca8 kind: Event lastTimestamp: "2026-07-02T19:11:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T19:11:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:11:34Z" name: splitter-graph-c41f9-5bc8d4797c-wprsh.18be8ec2b37a7a1f namespace: kserve-ci-e2e-test resourceVersion: "30549" uid: 95d4e06a-1c22-480e-a6c7-728963a66aa9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-c41f9-5bc8d4797c namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: bfbbdebb-d7f4-46f2-8ac2-4a4c5cf29ccf kind: Event lastTimestamp: "2026-07-02T19:02:52Z" message: 'Created pod: splitter-graph-c41f9-5bc8d4797c-wprsh' metadata: creationTimestamp: "2026-07-02T19:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:52Z" name: splitter-graph-c41f9-5bc8d4797c.18be8e4ef5de2e27 namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: 061ed8d9-7058-4c41-a187-6c1d4f400f8a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-c41f9 namespace: kserve-ci-e2e-test resourceVersion: "26854" uid: 78f129d0-61d8-4fcc-8927-5ec0a612a7ed kind: Event lastTimestamp: "2026-07-02T19:02:52Z" message: Scaled up replica set splitter-graph-c41f9-5bc8d4797c from 0 to 1 metadata: creationTimestamp: "2026-07-02T19:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:52Z" name: splitter-graph-c41f9.18be8e4ef51a0b1a namespace: kserve-ci-e2e-test resourceVersion: "26857" uid: 5455ccf9-7592-4bdf-971c-d2c896ed13ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:52Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-c41f9 namespace: kserve-ci-e2e-test resourceVersion: "26872" uid: 8ca86a53-f21d-4074-9e2d-491e56740321 kind: Event lastTimestamp: "2026-07-02T19:02:52Z" message: InferenceGraph [splitter-graph-c41f9] is Ready metadata: creationTimestamp: "2026-07-02T19:02:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:02:52Z" name: splitter-graph-c41f9.18be8e4efa6d70f9 namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: 01ab6c47-6c47-4da5-8d36-6750e181291d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:03:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-c41f9 namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 233d079b-08c7-4563-a4d7-0b8c7fdb9d97 kind: Event lastTimestamp: "2026-07-02T19:03:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:03:22Z" name: splitter-graph-c41f9.18be8e527386fc6c namespace: kserve-ci-e2e-test resourceVersion: "27102" uid: b81b9edf-525f-422f-b295-c93bf26b8fa0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:03:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-c41f9 namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 233d079b-08c7-4563-a4d7-0b8c7fdb9d97 kind: Event lastTimestamp: "2026-07-02T19:03:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:03:22Z" name: splitter-graph-c41f9.18be8e5273def711 namespace: kserve-ci-e2e-test resourceVersion: "27103" uid: be6ac8fb-5ccc-40f7-bff5-680e0efb51e4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T19:01:41.974143Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c532a-76d7947565-k9nvb namespace: kserve-ci-e2e-test resourceVersion: "26039" uid: 61d3472c-ea65-43b2-beaf-387cdbb7ea44 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-c532a-76d7947565-k9nvb to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T19:01:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T19:01:41Z" name: splitter-graph-c532a-76d7947565-k9nvb.18be8e3e97fa0616 namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: f56554f1-582c-4525-a703-d4c9abbd7dd5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:42Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c532a-76d7947565-k9nvb namespace: kserve-ci-e2e-test resourceVersion: "26048" uid: 61d3472c-ea65-43b2-beaf-387cdbb7ea44 kind: Event lastTimestamp: "2026-07-02T19:01:42Z" message: Add eth0 [10.134.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T19:01:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T19:01:42Z" name: splitter-graph-c532a-76d7947565-k9nvb.18be8e3eb176cacb namespace: kserve-ci-e2e-test resourceVersion: "26061" uid: 0d67afde-de56-454c-9a5b-31eb72cd8d67 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c532a} kind: Pod name: splitter-graph-c532a-76d7947565-k9nvb namespace: kserve-ci-e2e-test resourceVersion: "26042" uid: 61d3472c-ea65-43b2-beaf-387cdbb7ea44 kind: Event lastTimestamp: "2026-07-02T19:01:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:75c3c56106a5b8ed3fd4cb951cdacfbe184ab3e990f6497776c0954efdfa10b2" already present on machine metadata: creationTimestamp: "2026-07-02T19:01:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:42Z" name: splitter-graph-c532a-76d7947565-k9nvb.18be8e3eb2be8fda namespace: kserve-ci-e2e-test resourceVersion: "26064" uid: dab161c3-0be8-4529-83f4-a6fcbee92b5f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c532a} kind: Pod name: splitter-graph-c532a-76d7947565-k9nvb namespace: kserve-ci-e2e-test resourceVersion: "26042" uid: 61d3472c-ea65-43b2-beaf-387cdbb7ea44 kind: Event lastTimestamp: "2026-07-02T19:01:42Z" message: 'Created container: splitter-graph-c532a' metadata: creationTimestamp: "2026-07-02T19:01:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:42Z" name: splitter-graph-c532a-76d7947565-k9nvb.18be8e3eb5afb1b0 namespace: kserve-ci-e2e-test resourceVersion: "26065" uid: 9d0c4925-7f24-4f62-9047-27451ae726ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c532a} kind: Pod name: splitter-graph-c532a-76d7947565-k9nvb namespace: kserve-ci-e2e-test resourceVersion: "26042" uid: 61d3472c-ea65-43b2-beaf-387cdbb7ea44 kind: Event lastTimestamp: "2026-07-02T19:01:42Z" message: Started container splitter-graph-c532a metadata: creationTimestamp: "2026-07-02T19:01:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:42Z" name: splitter-graph-c532a-76d7947565-k9nvb.18be8e3eb61b8b0f namespace: kserve-ci-e2e-test resourceVersion: "26066" uid: b232c900-067c-49a1-b321-521fbd28a495 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c532a} kind: Pod name: splitter-graph-c532a-76d7947565-k9nvb namespace: kserve-ci-e2e-test resourceVersion: "26042" uid: 61d3472c-ea65-43b2-beaf-387cdbb7ea44 kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: Stopping container splitter-graph-c532a metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:52Z" name: splitter-graph-c532a-76d7947565-k9nvb.18be8e40f11cee86 namespace: kserve-ci-e2e-test resourceVersion: "26165" uid: d6a99853-328e-4f45-b548-4b91dd5ab69c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T19:01:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c532a} kind: Pod name: splitter-graph-c532a-76d7947565-k9nvb namespace: kserve-ci-e2e-test resourceVersion: "26042" uid: 61d3472c-ea65-43b2-beaf-387cdbb7ea44 kind: Event lastTimestamp: "2026-07-02T19:02:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T19:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:02:19Z" name: splitter-graph-c532a-76d7947565-k9nvb.18be8e4166f35060 namespace: kserve-ci-e2e-test resourceVersion: "26526" uid: 59ceb8e4-17fa-4480-80bd-6f729f9ad675 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-c532a-76d7947565 namespace: kserve-ci-e2e-test resourceVersion: "26035" uid: 67dc624f-3d4a-4633-b323-feed6454e81c kind: Event lastTimestamp: "2026-07-02T19:01:41Z" message: 'Created pod: splitter-graph-c532a-76d7947565-k9nvb' metadata: creationTimestamp: "2026-07-02T19:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:41Z" name: splitter-graph-c532a-76d7947565.18be8e3e97713dc5 namespace: kserve-ci-e2e-test resourceVersion: "26041" uid: 6d5da665-af30-4494-b350-a59a7020b3ce reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-c532a namespace: kserve-ci-e2e-test resourceVersion: "26031" uid: f1066ef9-9deb-4d04-ad25-c61aeb1cfa68 kind: Event lastTimestamp: "2026-07-02T19:01:41Z" message: Scaled up replica set splitter-graph-c532a-76d7947565 from 0 to 1 metadata: creationTimestamp: "2026-07-02T19:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:41Z" name: splitter-graph-c532a.18be8e3e96b62cea namespace: kserve-ci-e2e-test resourceVersion: "26038" uid: 916cc2c6-6175-48eb-8733-bfcf5315a759 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-c532a namespace: kserve-ci-e2e-test resourceVersion: "26051" uid: 0e8df0d5-7b80-4938-8ad1-8e40772fc685 kind: Event lastTimestamp: "2026-07-02T19:01:42Z" message: InferenceGraph [splitter-graph-c532a] is Ready metadata: creationTimestamp: "2026-07-02T19:01:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:01:42Z" name: splitter-graph-c532a.18be8e3e9a7c314c namespace: kserve-ci-e2e-test resourceVersion: "26053" uid: a02e494a-ec1f-4367-af53-ba4bce21d49b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T19:01:21.984538Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7 namespace: kserve-ci-e2e-test resourceVersion: "25741" uid: 704b39e9-fa02-4592-8f70-01d5bd2bacbf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7 to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T19:01:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T19:01:21Z" name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7.18be8e39f080c4b1 namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 58f68e6d-83f4-4ad1-9994-65aa0a706d8a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7 namespace: kserve-ci-e2e-test resourceVersion: "25745" uid: 704b39e9-fa02-4592-8f70-01d5bd2bacbf kind: Event lastTimestamp: "2026-07-02T19:01:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-208dc-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T19:01:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:22Z" name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7.18be8e39fdf328f6 namespace: kserve-ci-e2e-test resourceVersion: "25789" uid: f3142c0f-4014-4280-8f36-bd5637155a2c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7 namespace: kserve-ci-e2e-test resourceVersion: "25752" uid: 704b39e9-fa02-4592-8f70-01d5bd2bacbf kind: Event lastTimestamp: "2026-07-02T19:01:23Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T19:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T19:01:23Z" name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7.18be8e3a2de24076 namespace: kserve-ci-e2e-test resourceVersion: "25816" uid: 97afdf37-67bc-45d1-ace2-f8c0b5df13e2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7 namespace: kserve-ci-e2e-test resourceVersion: "25745" uid: 704b39e9-fa02-4592-8f70-01d5bd2bacbf kind: Event lastTimestamp: "2026-07-02T19:01:23Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T19:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:23Z" name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7.18be8e3a2f243045 namespace: kserve-ci-e2e-test resourceVersion: "25818" uid: 65387509-80bc-4ac5-a0ac-5460679584ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7 namespace: kserve-ci-e2e-test resourceVersion: "25745" uid: 704b39e9-fa02-4592-8f70-01d5bd2bacbf kind: Event lastTimestamp: "2026-07-02T19:01:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T19:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:23Z" name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7.18be8e3a32b92892 namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: 9f637c19-cdad-4fe1-a824-a4e3715993af reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7 namespace: kserve-ci-e2e-test resourceVersion: "25745" uid: 704b39e9-fa02-4592-8f70-01d5bd2bacbf kind: Event lastTimestamp: "2026-07-02T19:01:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T19:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:23Z" name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7.18be8e3a33278c5d namespace: kserve-ci-e2e-test resourceVersion: "25820" uid: c02bd40f-ca8e-40b3-8479-2282470247d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7 namespace: kserve-ci-e2e-test resourceVersion: "25745" uid: 704b39e9-fa02-4592-8f70-01d5bd2bacbf kind: Event lastTimestamp: "2026-07-02T19:01:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T19:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:23Z" name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7.18be8e3a33323b11 namespace: kserve-ci-e2e-test resourceVersion: "25821" uid: 4351c294-8bb9-4aef-b003-f457637ed744 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7 namespace: kserve-ci-e2e-test resourceVersion: "25745" uid: 704b39e9-fa02-4592-8f70-01d5bd2bacbf kind: Event lastTimestamp: "2026-07-02T19:01:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T19:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:23Z" name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7.18be8e3a363f411a namespace: kserve-ci-e2e-test resourceVersion: "25822" uid: bbf61c39-8d00-473c-9928-a60252edf296 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7 namespace: kserve-ci-e2e-test resourceVersion: "25745" uid: 704b39e9-fa02-4592-8f70-01d5bd2bacbf kind: Event lastTimestamp: "2026-07-02T19:01:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:23Z" name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7.18be8e3a36aec949 namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: 40b869ae-078d-4959-81ec-67c5e52c31d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T19:01:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7 namespace: kserve-ci-e2e-test resourceVersion: "25745" uid: 704b39e9-fa02-4592-8f70-01d5bd2bacbf kind: Event lastTimestamp: "2026-07-02T19:02:00Z" message: 'Readiness probe failed: dial tcp 10.134.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T19:01:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:02:00Z" name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7.18be8e3a6499ec4c namespace: kserve-ci-e2e-test resourceVersion: "26370" uid: b5492ea0-e59a-4183-9895-8743742bb4a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:18:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7 namespace: kserve-ci-e2e-test resourceVersion: "25745" uid: 704b39e9-fa02-4592-8f70-01d5bd2bacbf kind: Event lastTimestamp: "2026-07-02T19:18:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T19:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:18:41Z" name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7.18be8f2bfd004b2c namespace: kserve-ci-e2e-test resourceVersion: "33475" uid: 71bc6b99-6551-4aab-964d-24bb9dc337dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:18:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7 namespace: kserve-ci-e2e-test resourceVersion: "25745" uid: 704b39e9-fa02-4592-8f70-01d5bd2bacbf kind: Event lastTimestamp: "2026-07-02T19:18:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:18:41Z" name: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7.18be8f2bfd00e947 namespace: kserve-ci-e2e-test resourceVersion: "33476" uid: 43cb7307-eedb-40ef-bca2-462a58614134 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-208dc-predictor-7b67d8c5b7 namespace: kserve-ci-e2e-test resourceVersion: "25725" uid: 38905dbb-517d-4540-b2f0-e7ba85638b1d kind: Event lastTimestamp: "2026-07-02T19:01:21Z" message: 'Created pod: success-200-isvc-208dc-predictor-7b67d8c5b7-lmdz7' metadata: creationTimestamp: "2026-07-02T19:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:21Z" name: success-200-isvc-208dc-predictor-7b67d8c5b7.18be8e39eff1c3a7 namespace: kserve-ci-e2e-test resourceVersion: "25744" uid: 43aee6ea-c8f2-4191-a06a-6a7b6d8dc02b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-208dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25718" uid: bfb348cb-8a0c-4d0f-8f09-2704012ab4ed kind: Event lastTimestamp: "2026-07-02T19:01:21Z" message: Scaled up replica set success-200-isvc-208dc-predictor-7b67d8c5b7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T19:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:21Z" name: success-200-isvc-208dc-predictor.18be8e39ee23a80a namespace: kserve-ci-e2e-test resourceVersion: "25728" uid: f0abe2e1-f37c-46f1-b6e3-74050af0954a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:01:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-208dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25724" uid: 5d80b675-45d8-4331-b0ed-625a9ea7e778 kind: Event lastTimestamp: "2026-07-02T19:01:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T19:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:51Z" name: success-200-isvc-208dc-predictor.18be8e3d6cc8d9b7 namespace: kserve-ci-e2e-test resourceVersion: "26144" uid: 049e62d9-6011-4447-8160-1e53e0142b0d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:01:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-208dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25724" uid: 5d80b675-45d8-4331-b0ed-625a9ea7e778 kind: Event lastTimestamp: "2026-07-02T19:01:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T19:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:51Z" name: success-200-isvc-208dc-predictor.18be8e3d6d14b875 namespace: kserve-ci-e2e-test resourceVersion: "26147" uid: 1a288234-0a22-4dc4-8fd7-0fd73c10613b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:02:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-208dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25976" uid: 5d80b675-45d8-4331-b0ed-625a9ea7e778 kind: Event lastTimestamp: "2026-07-02T19:02:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:22Z" name: success-200-isvc-208dc-predictor.18be8e446a818b0d namespace: kserve-ci-e2e-test resourceVersion: "26553" uid: e5e88022-c273-4421-97e9-283a38d2478c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:02:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-208dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "25976" uid: 5d80b675-45d8-4331-b0ed-625a9ea7e778 kind: Event lastTimestamp: "2026-07-02T19:02:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:02:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:22Z" name: success-200-isvc-208dc-predictor.18be8e446af85617 namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: 3f1866ac-8024-42b6-a3e3-8b64a79325b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T19:01:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-208dc namespace: kserve-ci-e2e-test resourceVersion: "25695" uid: 72c3b31f-6aa4-4a11-a171-74b3d0a71206 kind: Event lastTimestamp: "2026-07-02T19:02:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T19:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:02:11Z" name: success-200-isvc-208dc.18be8e39ee6d0792 namespace: kserve-ci-e2e-test resourceVersion: "26471" uid: 4a17deb7-97c1-4086-a66e-5dd975b0eb24 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-208dc namespace: kserve-ci-e2e-test resourceVersion: "25695" uid: 72c3b31f-6aa4-4a11-a171-74b3d0a71206 kind: Event lastTimestamp: "2026-07-02T19:01:21Z" message: 'Failed to update status for InferenceService "success-200-isvc-208dc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-208dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T19:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:01:21Z" name: success-200-isvc-208dc.18be8e39eeda318d namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 13a32688-fc51-4017-b39a-c34d513f9cf4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-208dc namespace: kserve-ci-e2e-test resourceVersion: "25695" uid: 72c3b31f-6aa4-4a11-a171-74b3d0a71206 kind: Event lastTimestamp: "2026-07-02T19:01:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-208dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T19:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:01:21Z" name: success-200-isvc-208dc.18be8e39eeda6ec4 namespace: kserve-ci-e2e-test resourceVersion: "25740" uid: 1fa814a6-5b48-40af-95a2-fa5cd584d4e3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-208dc namespace: kserve-ci-e2e-test resourceVersion: "26469" uid: 72c3b31f-6aa4-4a11-a171-74b3d0a71206 kind: Event lastTimestamp: "2026-07-02T19:02:11Z" message: InferenceService [success-200-isvc-208dc] is Ready metadata: creationTimestamp: "2026-07-02T19:02:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:02:11Z" name: success-200-isvc-208dc.18be8e455b7176dd namespace: kserve-ci-e2e-test resourceVersion: "26470" uid: 1c30a668-2c3b-4b50-8caa-89c2f5274f0b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:51:27.160759Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf namespace: kserve-ci-e2e-test resourceVersion: "20674" uid: fffc5bbc-f8c3-4ba0-839c-d20e01d598bb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7acc8-predictor-5896b7c7c-74fgf to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf.18be8daf723e2dbd namespace: kserve-ci-e2e-test resourceVersion: "20680" uid: 670ccdc2-ed2d-47a8-99ea-e26e7ed71887 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: fffc5bbc-f8c3-4ba0-839c-d20e01d598bb kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf.18be8daf8c522c30 namespace: kserve-ci-e2e-test resourceVersion: "20741" uid: 933a722e-6f3b-4b6f-8369-f5155dbb4ca5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: fffc5bbc-f8c3-4ba0-839c-d20e01d598bb kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf.18be8daf8db285e5 namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: 4c1e4e28-27d5-4f9d-8f3f-c12b6d341282 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: fffc5bbc-f8c3-4ba0-839c-d20e01d598bb kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf.18be8daf90a34902 namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: 0a2a55be-6097-4872-b453-d79d4cceb296 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: fffc5bbc-f8c3-4ba0-839c-d20e01d598bb kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf.18be8daf912fc0ec namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 965ca0a3-0145-4d19-8fef-eed5cf7d9b63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: fffc5bbc-f8c3-4ba0-839c-d20e01d598bb kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf.18be8daf91399501 namespace: kserve-ci-e2e-test resourceVersion: "20747" uid: d1fe06ca-f4ca-4f3d-ae42-d9c7e9bc712e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: fffc5bbc-f8c3-4ba0-839c-d20e01d598bb kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf.18be8daf942790c3 namespace: kserve-ci-e2e-test resourceVersion: "20748" uid: 9bc80a7b-9b6d-4ef2-aff7-2a296ab04301 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: fffc5bbc-f8c3-4ba0-839c-d20e01d598bb kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf.18be8daf949b966f namespace: kserve-ci-e2e-test resourceVersion: "20749" uid: 54a57d38-c03b-46e9-8edb-c85035ad6399 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:51:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: fffc5bbc-f8c3-4ba0-839c-d20e01d598bb kind: Event lastTimestamp: "2026-07-02T18:52:05Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:05Z" name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf.18be8dafea2eeb28 namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: e07bea10-53bf-44db-9e2e-0979d94b6141 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: fffc5bbc-f8c3-4ba0-839c-d20e01d598bb kind: Event lastTimestamp: "2026-07-02T19:00:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:41Z" name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf.18be8e309f965e2b namespace: kserve-ci-e2e-test resourceVersion: "25023" uid: d38f8083-20e9-4c68-9f7b-dff3558c21d9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: fffc5bbc-f8c3-4ba0-839c-d20e01d598bb kind: Event lastTimestamp: "2026-07-02T19:00:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:41Z" name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf.18be8e309f97f78c namespace: kserve-ci-e2e-test resourceVersion: "25025" uid: 0119e893-f41d-403c-8fbb-c5fb9c2d48d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: fffc5bbc-f8c3-4ba0-839c-d20e01d598bb kind: Event lastTimestamp: "2026-07-02T19:00:46Z" message: 'Readiness probe failed: Get "https://10.134.0.43:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-02T19:00:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:46Z" name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf.18be8e319a52e4bf namespace: kserve-ci-e2e-test resourceVersion: "25245" uid: 6356cf45-b073-44a0-8771-c3f42e007e7d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: fffc5bbc-f8c3-4ba0-839c-d20e01d598bb kind: Event lastTimestamp: "2026-07-02T19:00:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: i/o timeout' metadata: creationTimestamp: "2026-07-02T19:00:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:46Z" name: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf.18be8e319a9837dd namespace: kserve-ci-e2e-test resourceVersion: "25246" uid: 6c193b64-15df-4236-b06a-af29e4d7089a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7acc8-predictor-5896b7c7c namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 53997a7a-1664-4fb6-be21-d81fa24dfb3e kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: 'Created pod: success-200-isvc-7acc8-predictor-5896b7c7c-74fgf' metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-7acc8-predictor-5896b7c7c.18be8daf71a8b68e namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: 32819575-bb8f-4dc6-aef2-61f01ec6e00b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7acc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20655" uid: 02b99478-4e19-4fab-8ab2-3c3415ebdd63 kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: Scaled up replica set success-200-isvc-7acc8-predictor-5896b7c7c from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-7acc8-predictor.18be8daf6f79064a namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 28abe076-b9fd-452b-a933-30db373a2248 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:51:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7acc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: da07c1a1-eb04-4c60-a167-6059617fe5df kind: Event lastTimestamp: "2026-07-02T18:51:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:57Z" name: success-200-isvc-7acc8-predictor.18be8db2eddb3e39 namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: 0a2e2a51-4971-472c-9a42-8522296b56e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:51:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7acc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: da07c1a1-eb04-4c60-a167-6059617fe5df kind: Event lastTimestamp: "2026-07-02T18:51:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:57Z" name: success-200-isvc-7acc8-predictor.18be8db2ee3fe63d namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: a00e8352-a247-4b2c-9f9d-d77bf1486a6b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:52:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7acc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: da07c1a1-eb04-4c60-a167-6059617fe5df kind: Event lastTimestamp: "2026-07-02T18:52:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:42Z" name: success-200-isvc-7acc8-predictor.18be8db9ebb7bf46 namespace: kserve-ci-e2e-test resourceVersion: "21619" uid: c832066f-d066-42e9-933b-4942d5973dcc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:52:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7acc8-predictor namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: da07c1a1-eb04-4c60-a167-6059617fe5df kind: Event lastTimestamp: "2026-07-02T18:52:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:42Z" name: success-200-isvc-7acc8-predictor.18be8db9ec1be216 namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: b99c96e4-0c10-45f5-ba3a-0b22189109eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: 9f302813-a40e-4767-a238-f35295931142 kind: Event lastTimestamp: "2026-07-02T18:52:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:52:15Z" name: success-200-isvc-7acc8.18be8daf6f920106 namespace: kserve-ci-e2e-test resourceVersion: "21371" uid: 5155144d-db86-4c8e-8b0c-f387b003811c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: 9f302813-a40e-4767-a238-f35295931142 kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: 'Failed to update status for InferenceService "success-200-isvc-7acc8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7acc8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-7acc8.18be8daf7046a99a namespace: kserve-ci-e2e-test resourceVersion: "20669" uid: 2da1ed4b-df40-4ace-83e8-1ed30c4a9b30 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: 9f302813-a40e-4767-a238-f35295931142 kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7acc8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-7acc8.18be8daf7046e769 namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 8ed016ad-bf9e-40ba-8e42-00448be500f1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7acc8 namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: 9f302813-a40e-4767-a238-f35295931142 kind: Event lastTimestamp: "2026-07-02T18:52:15Z" message: InferenceService [success-200-isvc-7acc8] is Ready metadata: creationTimestamp: "2026-07-02T18:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:52:15Z" name: success-200-isvc-7acc8.18be8dbaa56b5054 namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: cc608c63-23c3-4dd3-a0cb-1af33f0b5679 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:50:56.728135Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: 898521dc-c12d-49ba-91bb-d9bf1437cf21 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-94e8c-predictor-654cd696d7-pjsrs to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:50:56Z" name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs.18be8da85c51430a namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 6b1ecfc6-3902-490d-9171-0ff91e9cf782 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 898521dc-c12d-49ba-91bb-d9bf1437cf21 kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-94e8c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:56Z" name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs.18be8da869814070 namespace: kserve-ci-e2e-test resourceVersion: "20276" uid: f9bf6217-269b-4d3c-b5a0-9dca15e9dd6e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs namespace: kserve-ci-e2e-test resourceVersion: "20229" uid: 898521dc-c12d-49ba-91bb-d9bf1437cf21 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:50:57Z" name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs.18be8da899d7064d namespace: kserve-ci-e2e-test resourceVersion: "20290" uid: 47bbb875-debb-4930-bbe5-db1be89a4e02 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 898521dc-c12d-49ba-91bb-d9bf1437cf21 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:57Z" name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs.18be8da89b17c246 namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: 6a098009-f61d-4a68-9b6f-a9442c1da46e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 898521dc-c12d-49ba-91bb-d9bf1437cf21 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:57Z" name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs.18be8da89e162909 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 3a4994f1-5954-49ad-824a-3e2be5da87d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 898521dc-c12d-49ba-91bb-d9bf1437cf21 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:57Z" name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs.18be8da89eb7b63f namespace: kserve-ci-e2e-test resourceVersion: "20295" uid: 75a61ddd-dab1-44e4-9524-97bae203a92f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 898521dc-c12d-49ba-91bb-d9bf1437cf21 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:57Z" name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs.18be8da89ec16cb7 namespace: kserve-ci-e2e-test resourceVersion: "20296" uid: 0da12a1c-21fc-4f6b-bcda-8ab3d6669037 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 898521dc-c12d-49ba-91bb-d9bf1437cf21 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:57Z" name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs.18be8da8a218ef04 namespace: kserve-ci-e2e-test resourceVersion: "20299" uid: a9313d86-df1a-4ef4-9bd5-cb4ca816fc7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 898521dc-c12d-49ba-91bb-d9bf1437cf21 kind: Event lastTimestamp: "2026-07-02T18:50:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:57Z" name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs.18be8da8a28af72b namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: c3b327fc-dbc2-4ece-a811-8d19c26738fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:50:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 898521dc-c12d-49ba-91bb-d9bf1437cf21 kind: Event lastTimestamp: "2026-07-02T18:51:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:35Z" name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs.18be8da8e4f45a68 namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: 45a1794d-862a-4e78-bb19-080b84666bef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 898521dc-c12d-49ba-91bb-d9bf1437cf21 kind: Event lastTimestamp: "2026-07-02T18:52:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T18:52:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:06Z" name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs.18be8db8b6e282ec namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: b08c4e97-0fe1-49cd-8b79-f859632ab2a6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 898521dc-c12d-49ba-91bb-d9bf1437cf21 kind: Event lastTimestamp: "2026-07-02T18:52:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:52:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:06Z" name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs.18be8db8b6e2e5a5 namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: 09e7edcf-ec4f-4a71-a536-5e1b7ab8a689 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 898521dc-c12d-49ba-91bb-d9bf1437cf21 kind: Event lastTimestamp: "2026-07-02T18:52:10Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:52:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:10Z" name: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs.18be8db96d32a277 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: bf90c0ab-f178-4a6a-aacf-1b82ff60a9c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-94e8c-predictor-654cd696d7 namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: f33acc5f-1aef-4cf3-a8f6-f03304a63b9c kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: 'Created pod: success-200-isvc-94e8c-predictor-654cd696d7-pjsrs' metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:50:56Z" name: success-200-isvc-94e8c-predictor-654cd696d7.18be8da85bc3b5c1 namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: c4023c2e-a068-48a2-9a1e-4784d53e631f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-94e8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 8b32d1a2-6e24-4fd9-96af-66e13f8dec8e kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: Scaled up replica set success-200-isvc-94e8c-predictor-654cd696d7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:50:56Z" name: success-200-isvc-94e8c-predictor.18be8da859db89c0 namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: 3ca4f7cb-f31c-447b-8c14-c612ef8b9148 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:51:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-94e8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20202" uid: 4993941d-ac79-4d5f-a320-066bbc780363 kind: Event lastTimestamp: "2026-07-02T18:51:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:26Z" name: success-200-isvc-94e8c-predictor.18be8dabd85434d4 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: dc36ac68-f511-4b37-b217-6cbf2fe1f03f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:51:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-94e8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20202" uid: 4993941d-ac79-4d5f-a320-066bbc780363 kind: Event lastTimestamp: "2026-07-02T18:51:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:51:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:26Z" name: success-200-isvc-94e8c-predictor.18be8dabd8aedd05 namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 8b87c8b8-c685-47ce-8e2c-3f8afd8dff6a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:51:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-94e8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 4993941d-ac79-4d5f-a320-066bbc780363 kind: Event lastTimestamp: "2026-07-02T18:51:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:56Z" name: success-200-isvc-94e8c-predictor.18be8db2d68dbc5d namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: e3d23f6b-ed1c-4cac-bb0a-45895cea9c05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:51:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-94e8c-predictor namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 4993941d-ac79-4d5f-a320-066bbc780363 kind: Event lastTimestamp: "2026-07-02T18:51:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:51:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:56Z" name: success-200-isvc-94e8c-predictor.18be8db2d6f5acf2 namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: f20670fb-7050-481d-b085-137d11daeb4e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-94e8c namespace: kserve-ci-e2e-test resourceVersion: "20187" uid: 77d0c81d-d377-4174-b2e8-b208d7eec6d6 kind: Event lastTimestamp: "2026-07-02T18:51:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:45Z" name: success-200-isvc-94e8c.18be8da859e32638 namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: fc098db7-02b2-4a88-bfaa-70a4b1eb2a7c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-94e8c namespace: kserve-ci-e2e-test resourceVersion: "20187" uid: 77d0c81d-d377-4174-b2e8-b208d7eec6d6 kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: 'Failed to update status for InferenceService "success-200-isvc-94e8c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-94e8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:50:56Z" name: success-200-isvc-94e8c.18be8da85a502a98 namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: f54d99a5-c0eb-4b06-815d-bbe4c8e528a2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-94e8c namespace: kserve-ci-e2e-test resourceVersion: "20187" uid: 77d0c81d-d377-4174-b2e8-b208d7eec6d6 kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-94e8c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:50:56Z" name: success-200-isvc-94e8c.18be8da85a508247 namespace: kserve-ci-e2e-test resourceVersion: "20215" uid: 80adcb36-88be-49c6-a561-baebdc740c73 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-94e8c namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: 77d0c81d-d377-4174-b2e8-b208d7eec6d6 kind: Event lastTimestamp: "2026-07-02T18:51:45Z" message: InferenceService [success-200-isvc-94e8c] is Ready metadata: creationTimestamp: "2026-07-02T18:51:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:45Z" name: success-200-isvc-94e8c.18be8db39fcf0f3b namespace: kserve-ci-e2e-test resourceVersion: "20927" uid: f7c7db75-1862-4499-8ccb-706ad2019192 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:40:51.913944Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6 namespace: kserve-ci-e2e-test resourceVersion: "15298" uid: 3569485e-42cc-47cd-9239-125860130e89 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6 to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:40:51Z" name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6.18be8d1b8a9509f7 namespace: kserve-ci-e2e-test resourceVersion: "15308" uid: f91d3082-1e25-49b6-8011-9b3db20e8719 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6 namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: 3569485e-42cc-47cd-9239-125860130e89 kind: Event lastTimestamp: "2026-07-02T18:40:52Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:40:52Z" name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6.18be8d1ba459de04 namespace: kserve-ci-e2e-test resourceVersion: "15372" uid: 60b7f036-bdbb-45fd-aa1a-87b3b6fb2ffa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 3569485e-42cc-47cd-9239-125860130e89 kind: Event lastTimestamp: "2026-07-02T18:40:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T18:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:52Z" name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6.18be8d1ba5a42429 namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: fda6ef1a-e068-48a0-854b-569fbfa14bcc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 3569485e-42cc-47cd-9239-125860130e89 kind: Event lastTimestamp: "2026-07-02T18:40:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:52Z" name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6.18be8d1ba9f0a6e1 namespace: kserve-ci-e2e-test resourceVersion: "15377" uid: b8a05340-f7db-4f55-847f-db7784b17ba5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 3569485e-42cc-47cd-9239-125860130e89 kind: Event lastTimestamp: "2026-07-02T18:40:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:52Z" name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6.18be8d1baa57f9ab namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 5e3f12a5-9c27-4cbb-9e73-e7cff50d6612 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 3569485e-42cc-47cd-9239-125860130e89 kind: Event lastTimestamp: "2026-07-02T18:40:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T18:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:52Z" name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6.18be8d1baa63bcc6 namespace: kserve-ci-e2e-test resourceVersion: "15379" uid: e261fa0b-767b-4fd2-990f-2481198db14d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 3569485e-42cc-47cd-9239-125860130e89 kind: Event lastTimestamp: "2026-07-02T18:40:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:52Z" name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6.18be8d1bad503c71 namespace: kserve-ci-e2e-test resourceVersion: "15380" uid: 231b811b-aa8f-4246-8cf3-8df80dc9bc99 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 3569485e-42cc-47cd-9239-125860130e89 kind: Event lastTimestamp: "2026-07-02T18:40:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:40:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:52Z" name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6.18be8d1badb7426b namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: 6ad17c5b-f5a1-4adc-ac39-13b9794b0ec0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T18:40:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 3569485e-42cc-47cd-9239-125860130e89 kind: Event lastTimestamp: "2026-07-02T18:50:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:40:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:19Z" name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6.18be8d1bd5c45c0e namespace: kserve-ci-e2e-test resourceVersion: "19874" uid: df2da327-01a9-4192-885d-0eb8bec12f1f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 3569485e-42cc-47cd-9239-125860130e89 kind: Event lastTimestamp: "2026-07-02T18:50:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:16Z" name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6.18be8d9f16a77087 namespace: kserve-ci-e2e-test resourceVersion: "19783" uid: f26d2ced-5c57-48d1-a83a-3cf19e2e2c05 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 3569485e-42cc-47cd-9239-125860130e89 kind: Event lastTimestamp: "2026-07-02T18:50:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:16Z" name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6.18be8d9f16a8993b namespace: kserve-ci-e2e-test resourceVersion: "19786" uid: 55192e88-59d8-49db-abbc-76c75861e3c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6 namespace: kserve-ci-e2e-test resourceVersion: "15302" uid: 3569485e-42cc-47cd-9239-125860130e89 kind: Event lastTimestamp: "2026-07-02T18:50:19Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:50:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:19Z" name: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6.18be8d9f9e310e6c namespace: kserve-ci-e2e-test resourceVersion: "19873" uid: 1c82546f-2a82-4765-88c5-c1484eb890af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a5373-predictor-9c9cf6bb7 namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: a972f5db-eb94-444b-83d2-4f33de2ac104 kind: Event lastTimestamp: "2026-07-02T18:40:51Z" message: 'Created pod: success-200-isvc-a5373-predictor-9c9cf6bb7-5kvh6' metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:51Z" name: success-200-isvc-a5373-predictor-9c9cf6bb7.18be8d1b8a0860f9 namespace: kserve-ci-e2e-test resourceVersion: "15301" uid: 2bb0c266-7273-457b-bdb9-0166c0638f45 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a5373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15281" uid: 3ac576b1-4cce-4138-b48a-b4ce3c0f2b56 kind: Event lastTimestamp: "2026-07-02T18:40:51Z" message: Scaled up replica set success-200-isvc-a5373-predictor-9c9cf6bb7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:51Z" name: success-200-isvc-a5373-predictor.18be8d1b87063842 namespace: kserve-ci-e2e-test resourceVersion: "15284" uid: 0020cec7-fadd-46fc-8d6e-c7d360eace11 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:41:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a5373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 276f5888-fbd8-42a7-95ad-2896c5d169ca kind: Event lastTimestamp: "2026-07-02T18:41:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:41:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:21Z" name: success-200-isvc-a5373-predictor.18be8d1f05954260 namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: 13b3f0b4-2a9e-4e6e-953b-d4ecbde8f41f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:41:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a5373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: 276f5888-fbd8-42a7-95ad-2896c5d169ca kind: Event lastTimestamp: "2026-07-02T18:41:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:41:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:21Z" name: success-200-isvc-a5373-predictor.18be8d1f05fe72bc namespace: kserve-ci-e2e-test resourceVersion: "15671" uid: 3e4162ef-427b-47fc-9cd1-27dd4a9fdec8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:41:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a5373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 276f5888-fbd8-42a7-95ad-2896c5d169ca kind: Event lastTimestamp: "2026-07-02T18:42:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:41:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:06Z" name: success-200-isvc-a5373-predictor.18be8d260369d0d9 namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: d1fd7606-67c6-4193-9cd1-6e8bb0e8dbc6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:41:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a5373-predictor namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 276f5888-fbd8-42a7-95ad-2896c5d169ca kind: Event lastTimestamp: "2026-07-02T18:42:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:41:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:06Z" name: success-200-isvc-a5373-predictor.18be8d2603cc990d namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: d3d4928f-7ad7-413e-87b1-f460951bcb73 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T18:40:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a5373 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: c4399515-ac68-4015-b232-f2f0309403ce kind: Event lastTimestamp: "2026-07-02T18:41:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:41:39Z" name: success-200-isvc-a5373.18be8d1b871921fc namespace: kserve-ci-e2e-test resourceVersion: "15855" uid: fc64b623-7a51-4bf9-9e60-3172835dfa21 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a5373 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: c4399515-ac68-4015-b232-f2f0309403ce kind: Event lastTimestamp: "2026-07-02T18:40:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-a5373": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a5373": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:40:51Z" name: success-200-isvc-a5373.18be8d1b87993a45 namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: 8de6f13a-b60c-43a1-9491-460e64bc4900 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a5373 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: c4399515-ac68-4015-b232-f2f0309403ce kind: Event lastTimestamp: "2026-07-02T18:40:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a5373": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:40:51Z" name: success-200-isvc-a5373.18be8d1b879982d2 namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 3c8427c7-e960-4994-9697-7db44f16c5e3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a5373 namespace: kserve-ci-e2e-test resourceVersion: "15853" uid: c4399515-ac68-4015-b232-f2f0309403ce kind: Event lastTimestamp: "2026-07-02T18:41:39Z" message: InferenceService [success-200-isvc-a5373] is Ready metadata: creationTimestamp: "2026-07-02T18:41:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:41:39Z" name: success-200-isvc-a5373.18be8d26906bb938 namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 0ad2e71e-de98-466a-9794-45660edd8b1d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:52:06.986179Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: 533fd46b-98d4-4392-975d-79511afd23fb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ae135-predictor-686c9c7795-6p9jz to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:52:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:52:06Z" name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz.18be8db8b805f228 namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: 99911a51-d911-40ac-b1a7-c7bd5d1aadb6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 533fd46b-98d4-4392-975d-79511afd23fb kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:52:07Z" name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz.18be8db8d201bcc7 namespace: kserve-ci-e2e-test resourceVersion: "21254" uid: 1a0234c2-7017-4e2b-bca6-c02714038c77 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 533fd46b-98d4-4392-975d-79511afd23fb kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:07Z" name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz.18be8db8d34e3a54 namespace: kserve-ci-e2e-test resourceVersion: "21256" uid: 168f034b-9580-495d-b96d-ed4065bdaf4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 533fd46b-98d4-4392-975d-79511afd23fb kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:07Z" name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz.18be8db8d661ebdd namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: fbb9e6b0-6c6a-450b-b209-f2e0dd52d2c9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 533fd46b-98d4-4392-975d-79511afd23fb kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:07Z" name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz.18be8db8d6d272c4 namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: 3c00542e-d403-43c0-95ef-40774d44a1f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 533fd46b-98d4-4392-975d-79511afd23fb kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:07Z" name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz.18be8db8d6ddf583 namespace: kserve-ci-e2e-test resourceVersion: "21260" uid: 5002ce33-9328-443f-a0a1-6b8f590e1a16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 533fd46b-98d4-4392-975d-79511afd23fb kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:07Z" name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz.18be8db8dad2a72d namespace: kserve-ci-e2e-test resourceVersion: "21261" uid: afb23bc6-92a8-4b99-b2b1-8a3f74ea6146 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 533fd46b-98d4-4392-975d-79511afd23fb kind: Event lastTimestamp: "2026-07-02T18:52:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:52:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:07Z" name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz.18be8db8db4b0cd0 namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 613ef33c-8370-466e-a680-0434cfc87095 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:52:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 533fd46b-98d4-4392-975d-79511afd23fb kind: Event lastTimestamp: "2026-07-02T18:52:45Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:52:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:52:45Z" name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz.18be8db9469901cf namespace: kserve-ci-e2e-test resourceVersion: "21650" uid: 257a316c-468d-4656-80a7-1f0c21b37a5b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 533fd46b-98d4-4392-975d-79511afd23fb kind: Event lastTimestamp: "2026-07-02T19:01:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T19:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:21Z" name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz.18be8e39ef51ce2a namespace: kserve-ci-e2e-test resourceVersion: "25739" uid: b0851b9a-b31e-4c8a-97c3-7bfc43fd697f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz namespace: kserve-ci-e2e-test resourceVersion: "21195" uid: 533fd46b-98d4-4392-975d-79511afd23fb kind: Event lastTimestamp: "2026-07-02T19:01:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:01:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:21Z" name: success-200-isvc-ae135-predictor-686c9c7795-6p9jz.18be8e39ef52553e namespace: kserve-ci-e2e-test resourceVersion: "25742" uid: ef15e0a9-b0c4-466c-b0d4-1b7cc3aeb1ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ae135-predictor-686c9c7795 namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: 27a5f8ac-f488-4b3e-9d72-b8be38060a0c kind: Event lastTimestamp: "2026-07-02T18:52:06Z" message: 'Created pod: success-200-isvc-ae135-predictor-686c9c7795-6p9jz' metadata: creationTimestamp: "2026-07-02T18:52:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:06Z" name: success-200-isvc-ae135-predictor-686c9c7795.18be8db8b77d922f namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: ffbd719f-f92a-4dc2-b694-16eb46286bc0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ae135-predictor namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: c64163a2-d5ee-4dd2-954d-2772e4835bb3 kind: Event lastTimestamp: "2026-07-02T18:52:06Z" message: Scaled up replica set success-200-isvc-ae135-predictor-686c9c7795 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:52:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:06Z" name: success-200-isvc-ae135-predictor.18be8db8b55f4957 namespace: kserve-ci-e2e-test resourceVersion: "21178" uid: b6bd0842-e638-4a6b-a8d7-dd9895e2d58d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:52:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ae135-predictor namespace: kserve-ci-e2e-test resourceVersion: "21175" uid: 50823e7a-e4fa-4156-a5df-4a0cfb0caebe kind: Event lastTimestamp: "2026-07-02T18:52:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:52:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:36Z" name: success-200-isvc-ae135-predictor.18be8dbc340819d4 namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: 96fb1e3e-9e64-4e1a-8081-d8124a0fb5d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:52:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ae135-predictor namespace: kserve-ci-e2e-test resourceVersion: "21175" uid: 50823e7a-e4fa-4156-a5df-4a0cfb0caebe kind: Event lastTimestamp: "2026-07-02T18:52:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:52:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:52:36Z" name: success-200-isvc-ae135-predictor.18be8dbc34490d9a namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 3cae6cdf-f9fe-4f96-9c67-e43c0ac8f4e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:52:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ae135-predictor namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 50823e7a-e4fa-4156-a5df-4a0cfb0caebe kind: Event lastTimestamp: "2026-07-02T18:53:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:52:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:53:22Z" name: success-200-isvc-ae135-predictor.18be8dc331a4cf06 namespace: kserve-ci-e2e-test resourceVersion: "21971" uid: 244d697a-9414-4b6c-8f5b-d2ce5ebe2032 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:52:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ae135-predictor namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 50823e7a-e4fa-4156-a5df-4a0cfb0caebe kind: Event lastTimestamp: "2026-07-02T18:53:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:52:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:53:22Z" name: success-200-isvc-ae135-predictor.18be8dc3320237f7 namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: e56db17a-e1af-409f-9140-e2a0cb7ca15f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T18:52:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ae135 namespace: kserve-ci-e2e-test resourceVersion: "21162" uid: b6de4e25-a76d-4ae5-8bf9-dc946c8b1740 kind: Event lastTimestamp: "2026-07-02T18:52:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:52:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:52:55Z" name: success-200-isvc-ae135.18be8db8b5a571dc namespace: kserve-ci-e2e-test resourceVersion: "21742" uid: 2c9410b4-4d78-41ac-aec1-c200e24613ea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:52:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ae135 namespace: kserve-ci-e2e-test resourceVersion: "21740" uid: b6de4e25-a76d-4ae5-8bf9-dc946c8b1740 kind: Event lastTimestamp: "2026-07-02T18:52:55Z" message: InferenceService [success-200-isvc-ae135] is Ready metadata: creationTimestamp: "2026-07-02T18:52:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:52:55Z" name: success-200-isvc-ae135.18be8dc401d6b56a namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: 887c3153-2b65-4570-b9e3-8630c3615639 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:39:11.507927Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14166" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bb242-predictor-86b56f46d9-98smf to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:39:11Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d0429eac787 namespace: kserve-ci-e2e-test resourceVersion: "14173" uid: df4484c2-6663-47e8-9500-805fc093ca16 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14176" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: "2026-07-02T18:39:11Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:39:11Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d0443fe90c2 namespace: kserve-ci-e2e-test resourceVersion: "14253" uid: 7b1906f7-f0a4-49c8-b2e7-451e0adcc440 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: "2026-07-02T18:39:11Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1698" metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:11Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d0445482479 namespace: kserve-ci-e2e-test resourceVersion: "14257" uid: 835a7cf4-50f2-4909-a975-4aae0f9ee1f6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: "2026-07-02T18:39:25Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1698" in 13.996s (13.996s including waiting). Image size: 1337149534 bytes.' metadata: creationTimestamp: "2026-07-02T18:39:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:25Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d078786fe62 namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 22d11d4f-93aa-40d0-9859-e74d3ff48ec6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: "2026-07-02T18:39:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:26Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d078ab520ec namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: a76c3f96-5d3c-425b-80cb-7216cf641664 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: "2026-07-02T18:39:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:26Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d078b2baef2 namespace: kserve-ci-e2e-test resourceVersion: "14461" uid: aa6d8c76-b8fa-4cd8-ae20-b1b203afea97 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: "2026-07-02T18:39:26Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T18:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:26Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d078b3a2e32 namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: 34cd9e4f-6593-4fb7-9968-b5afc317f351 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: "2026-07-02T18:39:28Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.322s (2.322s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T18:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:28Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d0815ac5a8d namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: d657fedc-90a7-4224-970a-a442d2aaeacd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: "2026-07-02T18:39:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:28Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d08193fed47 namespace: kserve-ci-e2e-test resourceVersion: "14495" uid: 57b10278-a7ff-475c-9555-01cdc8d59462 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: "2026-07-02T18:39:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:39:28Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d0819b53965 namespace: kserve-ci-e2e-test resourceVersion: "14496" uid: 4ccae581-6830-4a65-b80a-f3acf1a350b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T18:39:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: "2026-07-02T18:40:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:39:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:54Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d082ec30c23 namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: 123d6ae5-4e44-4b82-b980-cf2ccf8fe479 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: "2026-07-02T18:40:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:51Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d1b8b97dc5a namespace: kserve-ci-e2e-test resourceVersion: "15317" uid: 8dd1c1b5-5e2a-4ba0-a978-a01627856bd5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: "2026-07-02T18:40:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:51Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d1b8b982929 namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: ae0d4e6b-39d8-492b-839e-5e9c3b589e1e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bb242-predictor-86b56f46d9-98smf namespace: kserve-ci-e2e-test resourceVersion: "14169" uid: 46ca39a8-4059-437d-ac65-46df16fd1679 kind: Event lastTimestamp: "2026-07-02T18:40:54Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:40:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:54Z" name: success-200-isvc-bb242-predictor-86b56f46d9-98smf.18be8d1c350ecee4 namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 23edbf45-87bc-4ec3-8f0d-4bc62783571f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bb242-predictor-86b56f46d9 namespace: kserve-ci-e2e-test resourceVersion: "14160" uid: d5fb082c-f7db-4038-83ad-ced10985d670 kind: Event lastTimestamp: "2026-07-02T18:39:11Z" message: 'Created pod: success-200-isvc-bb242-predictor-86b56f46d9-98smf' metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:11Z" name: success-200-isvc-bb242-predictor-86b56f46d9.18be8d042965e2e6 namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 4d696930-d5a4-4037-a5dc-4dce90211e9b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bb242-predictor namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: 5e6dbae6-24b0-4310-96bc-0103420bb4fa kind: Event lastTimestamp: "2026-07-02T18:39:11Z" message: Scaled up replica set success-200-isvc-bb242-predictor-86b56f46d9 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:11Z" name: success-200-isvc-bb242-predictor.18be8d0425e5d56c namespace: kserve-ci-e2e-test resourceVersion: "14161" uid: f4a9f359-b4ce-4d74-a54b-a1a68a5c02ef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bb242-predictor namespace: kserve-ci-e2e-test resourceVersion: "14178" uid: ac1aee8a-2b29-4f3f-a0f1-c366d8ada871 kind: Event lastTimestamp: "2026-07-02T18:39:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:56Z" name: success-200-isvc-bb242-predictor.18be8d07ab356089 namespace: kserve-ci-e2e-test resourceVersion: "14758" uid: 5511c8e0-2c03-4dc5-aa99-ced80a35cdb7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bb242-predictor namespace: kserve-ci-e2e-test resourceVersion: "14178" uid: ac1aee8a-2b29-4f3f-a0f1-c366d8ada871 kind: Event lastTimestamp: "2026-07-02T18:39:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:39:56Z" name: success-200-isvc-bb242-predictor.18be8d07ab95b288 namespace: kserve-ci-e2e-test resourceVersion: "14759" uid: 034c097e-a5a9-4fd7-b64f-2182d1204cef reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:40:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bb242-predictor namespace: kserve-ci-e2e-test resourceVersion: "14469" uid: ac1aee8a-2b29-4f3f-a0f1-c366d8ada871 kind: Event lastTimestamp: "2026-07-02T18:40:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:41Z" name: success-200-isvc-bb242-predictor.18be8d1228124a0b namespace: kserve-ci-e2e-test resourceVersion: "15153" uid: c08bfb61-f647-4e02-a41f-96113824fe8e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:40:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bb242-predictor namespace: kserve-ci-e2e-test resourceVersion: "14469" uid: ac1aee8a-2b29-4f3f-a0f1-c366d8ada871 kind: Event lastTimestamp: "2026-07-02T18:40:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:41Z" name: success-200-isvc-bb242-predictor.18be8d1228787f5f namespace: kserve-ci-e2e-test resourceVersion: "15155" uid: 8dc1ada2-7cea-43e2-bcfb-13e607189081 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T18:39:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bb242 namespace: kserve-ci-e2e-test resourceVersion: "14151" uid: a139445c-d2cc-4871-ad95-3d13771eb4da kind: Event lastTimestamp: "2026-07-02T18:40:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:40:14Z" name: success-200-isvc-bb242.18be8d042bf61365 namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: ea0e2885-0685-4f3f-8dd5-eb4476e99943 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bb242 namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: a139445c-d2cc-4871-ad95-3d13771eb4da kind: Event lastTimestamp: "2026-07-02T18:40:14Z" message: InferenceService [success-200-isvc-bb242] is Ready metadata: creationTimestamp: "2026-07-02T18:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:40:14Z" name: success-200-isvc-bb242.18be8d12e9e12163 namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: c2979f2d-d4b8-4941-8e9c-e0fb05a7402b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T19:01:52.200623Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt namespace: kserve-ci-e2e-test resourceVersion: "26204" uid: 29f316cb-9604-4d0e-b80e-a5c151e307ae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt.18be8e40f985aaf0 namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 6abcdafe-6afa-4dc5-b9b7-5534b93cbddf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt namespace: kserve-ci-e2e-test resourceVersion: "26216" uid: 29f316cb-9604-4d0e-b80e-a5c151e307ae kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: Add eth0 [10.134.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt.18be8e41135879cb namespace: kserve-ci-e2e-test resourceVersion: "26270" uid: 25228d95-32b0-41a8-a865-b7985ba2cd55 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 29f316cb-9604-4d0e-b80e-a5c151e307ae kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt.18be8e4114b58a08 namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 20523be3-dea1-4088-859b-1207599bcb64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 29f316cb-9604-4d0e-b80e-a5c151e307ae kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt.18be8e411798d5df namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: 14ef51ec-c865-4e2e-b0e5-2c5211a27a25 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 29f316cb-9604-4d0e-b80e-a5c151e307ae kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt.18be8e411829d6b1 namespace: kserve-ci-e2e-test resourceVersion: "26276" uid: 8a586660-a980-4e71-a973-b99d8fb53355 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 29f316cb-9604-4d0e-b80e-a5c151e307ae kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt.18be8e411832e45a namespace: kserve-ci-e2e-test resourceVersion: "26277" uid: e01c945d-b5c5-4cab-820b-c05b50be7f32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 29f316cb-9604-4d0e-b80e-a5c151e307ae kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt.18be8e411bb4e864 namespace: kserve-ci-e2e-test resourceVersion: "26278" uid: fcc9ea30-7834-41c0-8938-c148aae2e423 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 29f316cb-9604-4d0e-b80e-a5c151e307ae kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt.18be8e411c29f830 namespace: kserve-ci-e2e-test resourceVersion: "26279" uid: ef38d71b-69c3-4316-a44f-29e82489fc6a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T19:01:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 29f316cb-9604-4d0e-b80e-a5c151e307ae kind: Event lastTimestamp: "2026-07-02T19:02:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T19:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:02:30Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt.18be8e416a50dc3b namespace: kserve-ci-e2e-test resourceVersion: "26655" uid: d86f1c63-ee80-445c-b846-5bbcde154a31 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 29f316cb-9604-4d0e-b80e-a5c151e307ae kind: Event lastTimestamp: "2026-07-02T19:11:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T19:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:11:07Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt.18be8ec22aaf68b5 namespace: kserve-ci-e2e-test resourceVersion: "30330" uid: deba3f93-364f-42a1-8809-154cb0c4635c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 29f316cb-9604-4d0e-b80e-a5c151e307ae kind: Event lastTimestamp: "2026-07-02T19:11:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:11:07Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt.18be8ec22ab04325 namespace: kserve-ci-e2e-test resourceVersion: "30336" uid: 5ee7543a-42d9-45e8-9be5-d19384b8ac3c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:11:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 29f316cb-9604-4d0e-b80e-a5c151e307ae kind: Event lastTimestamp: "2026-07-02T19:11:11Z" message: 'Readiness probe failed: Get "https://10.134.0.51:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-02T19:11:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:11:11Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt.18be8ec31a83ef95 namespace: kserve-ci-e2e-test resourceVersion: "30381" uid: 547454db-141e-4bbe-ae75-c589183788af reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:11:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 29f316cb-9604-4d0e-b80e-a5c151e307ae kind: Event lastTimestamp: "2026-07-02T19:11:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.51:8080: i/o timeout' metadata: creationTimestamp: "2026-07-02T19:11:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:11:11Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt.18be8ec31ad3ba44 namespace: kserve-ci-e2e-test resourceVersion: "30382" uid: aebfb894-a0ea-4ed6-a610-92906275836d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c41f9-predictor-bf7df5ddc namespace: kserve-ci-e2e-test resourceVersion: "26189" uid: ef94b4bd-50eb-49e8-8795-bc6d1d06a0e5 kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: 'Created pod: success-200-isvc-c41f9-predictor-bf7df5ddc-9gcbt' metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c41f9-predictor-bf7df5ddc.18be8e40f920ea58 namespace: kserve-ci-e2e-test resourceVersion: "26206" uid: 5a01d04f-d315-437f-bb52-f5a21b12a36d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c41f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: 34ca13ec-1ea6-411f-89c4-98410a17dcf4 kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: Scaled up replica set success-200-isvc-c41f9-predictor-bf7df5ddc from 0 to 1 metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c41f9-predictor.18be8e40f752e630 namespace: kserve-ci-e2e-test resourceVersion: "26192" uid: e371c41a-f23c-409b-a070-7ea28a0a3507 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:02:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c41f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 854ce034-ca0f-49df-a91d-bed436c1ff1c kind: Event lastTimestamp: "2026-07-02T19:02:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T19:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:22Z" name: success-200-isvc-c41f9-predictor.18be8e4475ef549a namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: d959df35-d716-45e9-8dfb-efbd8437f33b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:02:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c41f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: 854ce034-ca0f-49df-a91d-bed436c1ff1c kind: Event lastTimestamp: "2026-07-02T19:02:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T19:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:22Z" name: success-200-isvc-c41f9-predictor.18be8e4476528062 namespace: kserve-ci-e2e-test resourceVersion: "26584" uid: 34fee9a0-3773-4d8d-a76f-db27a8d684c5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:02:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c41f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26422" uid: 854ce034-ca0f-49df-a91d-bed436c1ff1c kind: Event lastTimestamp: "2026-07-02T19:02:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:52Z" name: success-200-isvc-c41f9-predictor.18be8e4b73880457 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: aa0779de-6ad5-4b6d-9c0d-b0c90b9bf2ba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:02:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c41f9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26422" uid: 854ce034-ca0f-49df-a91d-bed436c1ff1c kind: Event lastTimestamp: "2026-07-02T19:02:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:52Z" name: success-200-isvc-c41f9-predictor.18be8e4b73e1d922 namespace: kserve-ci-e2e-test resourceVersion: "26849" uid: fcb41f72-95b5-452c-81d5-e9f4660a3bae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c41f9 namespace: kserve-ci-e2e-test resourceVersion: "26175" uid: c3fb063c-7ab8-4bcb-8e77-671afd704199 kind: Event lastTimestamp: "2026-07-02T19:02:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:02:40Z" name: success-200-isvc-c41f9.18be8e40f7a6fce4 namespace: kserve-ci-e2e-test resourceVersion: "26751" uid: 9e711483-79e2-4812-aac4-2b21a3db7f70 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c41f9 namespace: kserve-ci-e2e-test resourceVersion: "26175" uid: c3fb063c-7ab8-4bcb-8e77-671afd704199 kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: 'Failed to update status for InferenceService "success-200-isvc-c41f9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c41f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c41f9.18be8e40f82438bd namespace: kserve-ci-e2e-test resourceVersion: "26201" uid: 38488b93-1939-4977-bf25-e9ba2c05de86 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c41f9 namespace: kserve-ci-e2e-test resourceVersion: "26175" uid: c3fb063c-7ab8-4bcb-8e77-671afd704199 kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c41f9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c41f9.18be8e40f824a408 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: c7428523-4c86-46b4-bf53-81b5d3a9aaf3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c41f9 namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: c3fb063c-7ab8-4bcb-8e77-671afd704199 kind: Event lastTimestamp: "2026-07-02T19:02:40Z" message: InferenceService [success-200-isvc-c41f9] is Ready metadata: creationTimestamp: "2026-07-02T19:02:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:02:40Z" name: success-200-isvc-c41f9.18be8e4c255d7b44 namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: 61df46e1-a95d-4f33-aa97-256de36af313 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T19:00:41.965532Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv namespace: kserve-ci-e2e-test resourceVersion: "25014" uid: d6dcad90-e414-454f-bf37-f2c6d826c0f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c532a-predictor-6988ddc98f-c4fgv to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T19:00:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T19:00:41Z" name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv.18be8e309f2f3df5 namespace: kserve-ci-e2e-test resourceVersion: "25021" uid: 2cf00171-6d60-413c-9ba2-95296e2284c9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: d6dcad90-e414-454f-bf37-f2c6d826c0f1 kind: Event lastTimestamp: "2026-07-02T19:00:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c532a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T19:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:42Z" name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv.18be8e30aab24589 namespace: kserve-ci-e2e-test resourceVersion: "25072" uid: e84c20f2-0f07-4a51-ab2e-5e74787e34e2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv namespace: kserve-ci-e2e-test resourceVersion: "25032" uid: d6dcad90-e414-454f-bf37-f2c6d826c0f1 kind: Event lastTimestamp: "2026-07-02T19:00:43Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T19:00:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T19:00:43Z" name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv.18be8e30dd3b60af namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: 2d1fd6d1-d281-4fd0-9e19-4595444f9ad5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: d6dcad90-e414-454f-bf37-f2c6d826c0f1 kind: Event lastTimestamp: "2026-07-02T19:00:43Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T19:00:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:43Z" name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv.18be8e30de79555b namespace: kserve-ci-e2e-test resourceVersion: "25094" uid: 84344470-59db-4fa8-9726-ab7e522c8892 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: d6dcad90-e414-454f-bf37-f2c6d826c0f1 kind: Event lastTimestamp: "2026-07-02T19:00:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T19:00:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:43Z" name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv.18be8e30e1baa9e3 namespace: kserve-ci-e2e-test resourceVersion: "25095" uid: f076f66b-4d07-4ae1-98c5-f990fe6f3e7a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: d6dcad90-e414-454f-bf37-f2c6d826c0f1 kind: Event lastTimestamp: "2026-07-02T19:00:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T19:00:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:43Z" name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv.18be8e30e22b8d18 namespace: kserve-ci-e2e-test resourceVersion: "25096" uid: 5bd17011-b36a-4cc8-a29e-349b555dea02 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: d6dcad90-e414-454f-bf37-f2c6d826c0f1 kind: Event lastTimestamp: "2026-07-02T19:00:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T19:00:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:43Z" name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv.18be8e30e236ab67 namespace: kserve-ci-e2e-test resourceVersion: "25097" uid: 82831403-9303-4a49-93ed-45727942542c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: d6dcad90-e414-454f-bf37-f2c6d826c0f1 kind: Event lastTimestamp: "2026-07-02T19:00:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T19:00:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:43Z" name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv.18be8e30e54d8f24 namespace: kserve-ci-e2e-test resourceVersion: "25099" uid: 2dc302df-b5ea-48b1-9599-1f1c244c1c23 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: d6dcad90-e414-454f-bf37-f2c6d826c0f1 kind: Event lastTimestamp: "2026-07-02T19:00:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:00:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:00:43Z" name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv.18be8e30e5c43515 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 967f2942-0070-46e5-b668-1f10d571aea6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T19:00:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: d6dcad90-e414-454f-bf37-f2c6d826c0f1 kind: Event lastTimestamp: "2026-07-02T19:01:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T19:00:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:19Z" name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv.18be8e310844f8a6 namespace: kserve-ci-e2e-test resourceVersion: "25662" uid: f3b8481b-d8ae-489a-a364-947d2fc0a360 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: d6dcad90-e414-454f-bf37-f2c6d826c0f1 kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv.18be8e40f8ab761c namespace: kserve-ci-e2e-test resourceVersion: "26203" uid: 87afe53e-271b-4199-964c-9e7765da54bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: d6dcad90-e414-454f-bf37-f2c6d826c0f1 kind: Event lastTimestamp: "2026-07-02T19:01:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T19:01:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:52Z" name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv.18be8e40f8ac31b3 namespace: kserve-ci-e2e-test resourceVersion: "26207" uid: 29e6a58e-cbf8-4e41-864d-90cae8cb1615 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv namespace: kserve-ci-e2e-test resourceVersion: "25017" uid: d6dcad90-e414-454f-bf37-f2c6d826c0f1 kind: Event lastTimestamp: "2026-07-02T19:01:54Z" message: 'Readiness probe failed: Get "https://10.134.0.48:8643/healthz": dial tcp 10.134.0.48:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T19:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:01:54Z" name: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv.18be8e41908db0d9 namespace: kserve-ci-e2e-test resourceVersion: "26313" uid: 5b5ceee0-bc0b-447c-ae24-903f07c8e94e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c532a-predictor-6988ddc98f namespace: kserve-ci-e2e-test resourceVersion: "24996" uid: 5bbdc998-6688-44c6-913a-2b1189b80a40 kind: Event lastTimestamp: "2026-07-02T19:00:41Z" message: 'Created pod: success-200-isvc-c532a-predictor-6988ddc98f-c4fgv' metadata: creationTimestamp: "2026-07-02T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:00:41Z" name: success-200-isvc-c532a-predictor-6988ddc98f.18be8e309ea1537a namespace: kserve-ci-e2e-test resourceVersion: "25016" uid: 6537fd3f-cf1a-469d-bfb7-9f47d8167a29 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:00:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c532a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24993" uid: 0051aed3-a27f-4b2c-b8a9-0c178a77cf53 kind: Event lastTimestamp: "2026-07-02T19:00:41Z" message: Scaled up replica set success-200-isvc-c532a-predictor-6988ddc98f from 0 to 1 metadata: creationTimestamp: "2026-07-02T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:00:41Z" name: success-200-isvc-c532a-predictor.18be8e309b654224 namespace: kserve-ci-e2e-test resourceVersion: "24997" uid: 65dbf819-aedb-4e2a-b846-ddf3f955a39d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:00:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c532a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24995" uid: cd6781ca-f173-41d9-836a-27aa6a10ebf2 kind: Event lastTimestamp: "2026-07-02T19:01:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T19:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:11Z" name: success-200-isvc-c532a-predictor.18be8e3419d81dd7 namespace: kserve-ci-e2e-test resourceVersion: "25598" uid: 396326ed-9806-40d7-ae35-c83ca44c49e0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:00:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c532a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24995" uid: cd6781ca-f173-41d9-836a-27aa6a10ebf2 kind: Event lastTimestamp: "2026-07-02T19:01:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T19:00:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:11Z" name: success-200-isvc-c532a-predictor.18be8e341a3641bd namespace: kserve-ci-e2e-test resourceVersion: "25599" uid: 1baf3c94-b4d8-4a1b-bc74-de0458911d54 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:01:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c532a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25483" uid: cd6781ca-f173-41d9-836a-27aa6a10ebf2 kind: Event lastTimestamp: "2026-07-02T19:01:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:41Z" name: success-200-isvc-c532a-predictor.18be8e3b17bc125e namespace: kserve-ci-e2e-test resourceVersion: "26037" uid: fbb955b1-c5e3-48df-9ffa-fffcce75a0a6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:01:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c532a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25483" uid: cd6781ca-f173-41d9-836a-27aa6a10ebf2 kind: Event lastTimestamp: "2026-07-02T19:01:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T19:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:01:41Z" name: success-200-isvc-c532a-predictor.18be8e3b181e6634 namespace: kserve-ci-e2e-test resourceVersion: "26040" uid: 5b39a2a9-e9a4-427a-a8ea-be4d57e78ee4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T19:00:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c532a namespace: kserve-ci-e2e-test resourceVersion: "24985" uid: 68a1ce93-77c1-4740-8586-d3f100d8ec00 kind: Event lastTimestamp: "2026-07-02T19:01:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:01:29Z" name: success-200-isvc-c532a.18be8e309b6d668e namespace: kserve-ci-e2e-test resourceVersion: "25916" uid: d453f900-9204-41f8-ac52-b8545649c452 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:01:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c532a namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: 68a1ce93-77c1-4740-8586-d3f100d8ec00 kind: Event lastTimestamp: "2026-07-02T19:01:29Z" message: InferenceService [success-200-isvc-c532a] is Ready metadata: creationTimestamp: "2026-07-02T19:01:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:01:29Z" name: success-200-isvc-c532a.18be8e3bc326a68a namespace: kserve-ci-e2e-test resourceVersion: "25915" uid: 29cdb16d-d72e-4f0b-828c-c6ece4bc614d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:50:16.889673Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv namespace: kserve-ci-e2e-test resourceVersion: "19765" uid: 104b64ce-542f-4113-9fea-d346c4f4991e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:50:16Z" name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv.18be8d9f15c27c8b namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: e8bcb808-c2c5-4939-9664-eddc642ea3fd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv namespace: kserve-ci-e2e-test resourceVersion: "19789" uid: 104b64ce-542f-4113-9fea-d346c4f4991e kind: Event lastTimestamp: "2026-07-02T18:50:17Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:50:17Z" name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv.18be8d9f2f5769ea namespace: kserve-ci-e2e-test resourceVersion: "19837" uid: 6ff66421-8ff3-4757-b7ad-741d32173037 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 104b64ce-542f-4113-9fea-d346c4f4991e kind: Event lastTimestamp: "2026-07-02T18:50:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T18:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:17Z" name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv.18be8d9f30babd68 namespace: kserve-ci-e2e-test resourceVersion: "19839" uid: 462a6e7a-5d83-4e8c-aeaf-3b455a2a5ec1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 104b64ce-542f-4113-9fea-d346c4f4991e kind: Event lastTimestamp: "2026-07-02T18:50:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:17Z" name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv.18be8d9f3372eb2f namespace: kserve-ci-e2e-test resourceVersion: "19841" uid: ebc53f6b-ef4d-4100-8d29-dd3d06d95bfe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 104b64ce-542f-4113-9fea-d346c4f4991e kind: Event lastTimestamp: "2026-07-02T18:50:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:17Z" name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv.18be8d9f33e2addf namespace: kserve-ci-e2e-test resourceVersion: "19842" uid: 2698b385-afb8-4c42-ad8d-d29c7492a66f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 104b64ce-542f-4113-9fea-d346c4f4991e kind: Event lastTimestamp: "2026-07-02T18:50:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T18:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:17Z" name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv.18be8d9f33ed658d namespace: kserve-ci-e2e-test resourceVersion: "19843" uid: 6005b28e-67e7-4f6d-a8ba-b8368312f96f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 104b64ce-542f-4113-9fea-d346c4f4991e kind: Event lastTimestamp: "2026-07-02T18:50:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:17Z" name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv.18be8d9f3841b643 namespace: kserve-ci-e2e-test resourceVersion: "19844" uid: ceacf26a-2114-4ed4-8752-0edb1a1d9d34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 104b64ce-542f-4113-9fea-d346c4f4991e kind: Event lastTimestamp: "2026-07-02T18:50:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:17Z" name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv.18be8d9f38b712ae namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: 47e7f0b4-426c-4fca-a10b-82fa296ce44a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:50:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 104b64ce-542f-4113-9fea-d346c4f4991e kind: Event lastTimestamp: "2026-07-02T18:50:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:53Z" name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv.18be8d9f4c70a46f namespace: kserve-ci-e2e-test resourceVersion: "20145" uid: 115f3eff-7e52-4075-8f3f-30f268e2b89b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 104b64ce-542f-4113-9fea-d346c4f4991e kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv.18be8daf709507bf namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 008e26a7-b20f-4666-8095-69ad47dec381 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 104b64ce-542f-4113-9fea-d346c4f4991e kind: Event lastTimestamp: "2026-07-02T18:51:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:51:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:27Z" name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv.18be8daf70958aac namespace: kserve-ci-e2e-test resourceVersion: "20673" uid: 6463ee95-697a-4e11-bef8-69a9ffb062ba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 104b64ce-542f-4113-9fea-d346c4f4991e kind: Event lastTimestamp: "2026-07-02T18:51:28Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:51:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:51:28Z" name: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv.18be8dafd4e798b7 namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 16baad51-7438-4617-9d73-fba489ff1658 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e22cf-predictor-7dd874c4cc namespace: kserve-ci-e2e-test resourceVersion: "19749" uid: b7fafaea-da91-44f9-8b1d-f3bae54ca5d8 kind: Event lastTimestamp: "2026-07-02T18:50:16Z" message: 'Created pod: success-200-isvc-e22cf-predictor-7dd874c4cc-fxtqv' metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:50:16Z" name: success-200-isvc-e22cf-predictor-7dd874c4cc.18be8d9f152b8d75 namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: ee2ceaf5-ab10-48e2-9f6c-6168c28b7ef1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e22cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19747" uid: a9bab8e3-b65e-4931-9513-5d4dc18468f2 kind: Event lastTimestamp: "2026-07-02T18:50:16Z" message: Scaled up replica set success-200-isvc-e22cf-predictor-7dd874c4cc from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:50:16Z" name: success-200-isvc-e22cf-predictor.18be8d9f119be715 namespace: kserve-ci-e2e-test resourceVersion: "19752" uid: 1a85df77-c6c7-4d8c-bc75-f6a77c95d4c5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e22cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 2103393d-c582-4d0f-83e2-a55ab1dfbf0d kind: Event lastTimestamp: "2026-07-02T18:50:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:50:31Z" name: success-200-isvc-e22cf-predictor.18be8da2902245e7 namespace: kserve-ci-e2e-test resourceVersion: "19976" uid: b99ced17-5edb-42c1-9519-f2b8d08fcdad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e22cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 2103393d-c582-4d0f-83e2-a55ab1dfbf0d kind: Event lastTimestamp: "2026-07-02T18:50:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:50:31Z" name: success-200-isvc-e22cf-predictor.18be8da2908473c5 namespace: kserve-ci-e2e-test resourceVersion: "19978" uid: 9a33ad4c-0dd2-4147-a7dc-694a6d5a0a36 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:50:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e22cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: 2103393d-c582-4d0f-83e2-a55ab1dfbf0d kind: Event lastTimestamp: "2026-07-02T18:51:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:16Z" name: success-200-isvc-e22cf-predictor.18be8da60f1f93fa namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 29b8b6b6-6a81-4720-92ce-a148b4c4f233 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:50:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e22cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: 2103393d-c582-4d0f-83e2-a55ab1dfbf0d kind: Event lastTimestamp: "2026-07-02T18:51:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:51:16Z" name: success-200-isvc-e22cf-predictor.18be8da60f5f3b98 namespace: kserve-ci-e2e-test resourceVersion: "20503" uid: 7d051050-1682-44ff-9782-b4446bd9886d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T18:50:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e22cf namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: 27d40341-829c-4fe1-a482-7773897f06b2 kind: Event lastTimestamp: "2026-07-02T18:51:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:03Z" name: success-200-isvc-e22cf.18be8d9f119efeac namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: 0239783d-0422-460e-9821-0557603576e6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e22cf namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: 27d40341-829c-4fe1-a482-7773897f06b2 kind: Event lastTimestamp: "2026-07-02T18:50:16Z" message: 'Failed to update status for InferenceService "success-200-isvc-e22cf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e22cf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:50:16Z" name: success-200-isvc-e22cf.18be8d9f12491b98 namespace: kserve-ci-e2e-test resourceVersion: "19754" uid: f40efdcf-a137-4b0b-b559-843dcc08d087 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e22cf namespace: kserve-ci-e2e-test resourceVersion: "19739" uid: 27d40341-829c-4fe1-a482-7773897f06b2 kind: Event lastTimestamp: "2026-07-02T18:50:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e22cf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:50:16Z" name: success-200-isvc-e22cf.18be8d9f1249644c namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: 2c7d41cd-d419-40fc-ba3b-5c2b3d8b2028 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:51:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e22cf namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: 27d40341-829c-4fe1-a482-7773897f06b2 kind: Event lastTimestamp: "2026-07-02T18:51:03Z" message: InferenceService [success-200-isvc-e22cf] is Ready metadata: creationTimestamp: "2026-07-02T18:51:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:51:03Z" name: success-200-isvc-e22cf.18be8daa07662f9e namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: 2661946c-6154-416d-ac7b-d95a661fcb69 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:41:41.817725Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: d23f82c5-0e33-4821-a596-783a6938c9db kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:41:41Z" name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d.18be8d2729145598 namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 3a7458ac-ddb7-4c23-9ba2-54b1096def58 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: d23f82c5-0e33-4821-a596-783a6938c9db kind: Event lastTimestamp: "2026-07-02T18:41:42Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:41:42Z" name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d.18be8d2743737594 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: 65b17ce1-0aee-4b4a-9467-67cf6673ad4f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: d23f82c5-0e33-4821-a596-783a6938c9db kind: Event lastTimestamp: "2026-07-02T18:41:42Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1698" already present on machine metadata: creationTimestamp: "2026-07-02T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:42Z" name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d.18be8d2744d1a5e3 namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: b3a071ff-c952-4ea8-ac43-d5f2c9972a1c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: d23f82c5-0e33-4821-a596-783a6938c9db kind: Event lastTimestamp: "2026-07-02T18:41:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:42Z" name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d.18be8d2747787f2f namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 3e6fbed2-cc55-41f1-a905-661ee1596dc9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: d23f82c5-0e33-4821-a596-783a6938c9db kind: Event lastTimestamp: "2026-07-02T18:41:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:42Z" name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d.18be8d274808228a namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 6cd5143e-5c17-40e4-911d-6a84f05df4f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: d23f82c5-0e33-4821-a596-783a6938c9db kind: Event lastTimestamp: "2026-07-02T18:41:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:42Z" name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d.18be8d27481258da namespace: kserve-ci-e2e-test resourceVersion: "16028" uid: f37ea2fb-e73a-4483-b328-938dee4e751d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: d23f82c5-0e33-4821-a596-783a6938c9db kind: Event lastTimestamp: "2026-07-02T18:41:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:42Z" name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d.18be8d274b69b7f8 namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: 180b6061-05a5-476e-9661-12292cdba5af reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: d23f82c5-0e33-4821-a596-783a6938c9db kind: Event lastTimestamp: "2026-07-02T18:41:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:42Z" name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d.18be8d274bd67f8a namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 964cbba2-1407-4138-9cef-41d3234e9971 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T18:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: d23f82c5-0e33-4821-a596-783a6938c9db kind: Event lastTimestamp: "2026-07-02T18:50:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:59Z" name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d.18be8d2788302b09 namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: ca6f9391-6924-4e00-b232-270bf84b48ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: d23f82c5-0e33-4821-a596-783a6938c9db kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:56Z" name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d.18be8da85a876de4 namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: 1a14b8ba-8463-418f-8678-cacd2bcd70c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: d23f82c5-0e33-4821-a596-783a6938c9db kind: Event lastTimestamp: "2026-07-02T18:50:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T18:50:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:56Z" name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d.18be8da85a881656 namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: 02f84545-6593-43ac-b7a2-38cdb8a8d5e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d namespace: kserve-ci-e2e-test resourceVersion: "15963" uid: d23f82c5-0e33-4821-a596-783a6938c9db kind: Event lastTimestamp: "2026-07-02T18:50:59Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T18:50:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:59Z" name: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d.18be8da8fcb67509 namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 9d64cce9-e1bf-469e-8332-887bd777ef9e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e8b76-predictor-66f687f9b9 namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 2c76e06f-b796-41f7-a0be-5ae3c1dd89cc kind: Event lastTimestamp: "2026-07-02T18:41:41Z" message: 'Created pod: success-200-isvc-e8b76-predictor-66f687f9b9-j5g9d' metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:41Z" name: success-200-isvc-e8b76-predictor-66f687f9b9.18be8d272888da37 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: 04274bf2-2582-48b1-b778-a79699470536 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e8b76-predictor namespace: kserve-ci-e2e-test resourceVersion: "15937" uid: d037f226-40ae-40be-a24c-9a8c3b2cc2fa kind: Event lastTimestamp: "2026-07-02T18:41:41Z" message: Scaled up replica set success-200-isvc-e8b76-predictor-66f687f9b9 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:41Z" name: success-200-isvc-e8b76-predictor.18be8d272705f7b9 namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: d0a4facf-111b-4f25-9f22-aef3389e3cfa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:41:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e8b76-predictor namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: c0ad104f-06ca-4eeb-bcbb-d944c51afae9 kind: Event lastTimestamp: "2026-07-02T18:42:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:26Z" name: success-200-isvc-e8b76-predictor.18be8d2aa5cb0e40 namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: 16ee704c-2a1d-4fa9-bf81-799684b031d5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T18:41:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e8b76-predictor namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: c0ad104f-06ca-4eeb-bcbb-d944c51afae9 kind: Event lastTimestamp: "2026-07-02T18:42:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:41:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:26Z" name: success-200-isvc-e8b76-predictor.18be8d2aa6258739 namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: 9c0f6a6f-4593-4e59-be1d-f92365e77ed5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e8b76-predictor namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: c0ad104f-06ca-4eeb-bcbb-d944c51afae9 kind: Event lastTimestamp: "2026-07-02T18:42:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:41Z" name: success-200-isvc-e8b76-predictor.18be8d35226be91b namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: 809f538d-52fd-4a19-80ba-f76a0b418d0d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e8b76-predictor namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: c0ad104f-06ca-4eeb-bcbb-d944c51afae9 kind: Event lastTimestamp: "2026-07-02T18:42:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T18:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:41Z" name: success-200-isvc-e8b76-predictor.18be8d3522e6dee0 namespace: kserve-ci-e2e-test resourceVersion: "16593" uid: e49199ec-be7b-484b-a13b-0028f53882b5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:41:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e8b76 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 12954d88-b828-49d4-a39e-d0c2c4f8d498 kind: Event lastTimestamp: "2026-07-02T18:42:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T18:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:42:29Z" name: success-200-isvc-e8b76.18be8d27270d6a1d namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: bc30c8cd-de81-4fbe-ba32-3aabfd1df527 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:42:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e8b76 namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: 12954d88-b828-49d4-a39e-d0c2c4f8d498 kind: Event lastTimestamp: "2026-07-02T18:42:29Z" message: InferenceService [success-200-isvc-e8b76] is Ready metadata: creationTimestamp: "2026-07-02T18:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:42:29Z" name: success-200-isvc-e8b76.18be8d3243040083 namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: ea0e222f-8cea-4b8a-8e11-388746e7e083 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T19:02:22.041014Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-208dc-64b8656f78-th996 namespace: kserve-ci-e2e-test resourceVersion: "26563" uid: ff90b279-e314-4f0a-a1e3-5c715a4a7e28 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-208dc-64b8656f78-th996 to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T19:02:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T19:02:22Z" name: switch-graph-208dc-64b8656f78-th996.18be8e47ec25dcee namespace: kserve-ci-e2e-test resourceVersion: "26568" uid: 0c751df4-c0ec-47a5-865a-b7bc6aef2a33 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:22Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-208dc-64b8656f78-th996 namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: ff90b279-e314-4f0a-a1e3-5c715a4a7e28 kind: Event lastTimestamp: "2026-07-02T19:02:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-208dc-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T19:02:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:02:22Z" name: switch-graph-208dc-64b8656f78-th996.18be8e47f626b9a1 namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: bebb84ed-d531-4daa-884c-1b0a683202d5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:23Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-208dc-64b8656f78-th996 namespace: kserve-ci-e2e-test resourceVersion: "26572" uid: ff90b279-e314-4f0a-a1e3-5c715a4a7e28 kind: Event lastTimestamp: "2026-07-02T19:02:23Z" message: Add eth0 [10.134.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T19:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T19:02:23Z" name: switch-graph-208dc-64b8656f78-th996.18be8e4829788184 namespace: kserve-ci-e2e-test resourceVersion: "26596" uid: 5ef3acc3-eae6-4eea-86ba-a5752c189af3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-208dc} kind: Pod name: switch-graph-208dc-64b8656f78-th996 namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: ff90b279-e314-4f0a-a1e3-5c715a4a7e28 kind: Event lastTimestamp: "2026-07-02T19:02:23Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:75c3c56106a5b8ed3fd4cb951cdacfbe184ab3e990f6497776c0954efdfa10b2" already present on machine metadata: creationTimestamp: "2026-07-02T19:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:02:23Z" name: switch-graph-208dc-64b8656f78-th996.18be8e482aaa58e2 namespace: kserve-ci-e2e-test resourceVersion: "26598" uid: fda76462-4a9d-419c-b861-6cc33f886c58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-208dc} kind: Pod name: switch-graph-208dc-64b8656f78-th996 namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: ff90b279-e314-4f0a-a1e3-5c715a4a7e28 kind: Event lastTimestamp: "2026-07-02T19:02:23Z" message: 'Created container: switch-graph-208dc' metadata: creationTimestamp: "2026-07-02T19:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:02:23Z" name: switch-graph-208dc-64b8656f78-th996.18be8e482da5b8d5 namespace: kserve-ci-e2e-test resourceVersion: "26599" uid: 9d7e7f4b-ec1b-4c79-940a-f839a1720526 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-208dc} kind: Pod name: switch-graph-208dc-64b8656f78-th996 namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: ff90b279-e314-4f0a-a1e3-5c715a4a7e28 kind: Event lastTimestamp: "2026-07-02T19:02:23Z" message: Started container switch-graph-208dc metadata: creationTimestamp: "2026-07-02T19:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:02:23Z" name: switch-graph-208dc-64b8656f78-th996.18be8e482e120961 namespace: kserve-ci-e2e-test resourceVersion: "26600" uid: 673208dc-7d6f-4745-9392-db0fd5f122ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:18:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-208dc} kind: Pod name: switch-graph-208dc-64b8656f78-th996 namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: ff90b279-e314-4f0a-a1e3-5c715a4a7e28 kind: Event lastTimestamp: "2026-07-02T19:18:41Z" message: Stopping container switch-graph-208dc metadata: creationTimestamp: "2026-07-02T19:18:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:18:41Z" name: switch-graph-208dc-64b8656f78-th996.18be8f2bf4b509c6 namespace: kserve-ci-e2e-test resourceVersion: "33459" uid: f2000a7b-bd34-4361-a9a2-f57577dd6dc6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T19:18:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-208dc} kind: Pod name: switch-graph-208dc-64b8656f78-th996 namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: ff90b279-e314-4f0a-a1e3-5c715a4a7e28 kind: Event lastTimestamp: "2026-07-02T19:19:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T19:18:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T19:19:05Z" name: switch-graph-208dc-64b8656f78-th996.18be8f2cd7a7c918 namespace: kserve-ci-e2e-test resourceVersion: "33708" uid: 415a43a0-e386-4e9c-8553-6f5e9951c967 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-208dc-64b8656f78 namespace: kserve-ci-e2e-test resourceVersion: "26559" uid: 8427817a-2b8e-4f1d-bb6d-2bf955c5068c kind: Event lastTimestamp: "2026-07-02T19:02:22Z" message: 'Created pod: switch-graph-208dc-64b8656f78-th996' metadata: creationTimestamp: "2026-07-02T19:02:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:22Z" name: switch-graph-208dc-64b8656f78.18be8e47eba2256f namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 083bc125-61f7-411e-82a4-b65442b5c1a3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-208dc namespace: kserve-ci-e2e-test resourceVersion: "26556" uid: 27439e34-a28e-4e63-8f99-7f91306dd390 kind: Event lastTimestamp: "2026-07-02T19:02:22Z" message: Scaled up replica set switch-graph-208dc-64b8656f78 from 0 to 1 metadata: creationTimestamp: "2026-07-02T19:02:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:22Z" name: switch-graph-208dc.18be8e47eae88085 namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: 09722df9-1ba1-4b63-aff0-0d41ff31572f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T19:02:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-208dc namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 7b863223-2413-4259-8dce-6f63f22d5ddb kind: Event lastTimestamp: "2026-07-02T19:02:22Z" message: InferenceGraph [switch-graph-208dc] is Ready metadata: creationTimestamp: "2026-07-02T19:02:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T19:02:22Z" name: switch-graph-208dc.18be8e47ee934271 namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: e0fced10-2cf3-4a0f-b8cf-3af8e0f3c4ce reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:02:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-208dc namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: 9905c63b-1cb2-4267-923b-80d142cd99b8 kind: Event lastTimestamp: "2026-07-02T19:02:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T19:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:52Z" name: switch-graph-208dc.18be8e4b692b9508 namespace: kserve-ci-e2e-test resourceVersion: "26840" uid: 00127bf9-e658-4344-8e2a-af6b598ad8ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T19:02:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-208dc namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: 9905c63b-1cb2-4267-923b-80d142cd99b8 kind: Event lastTimestamp: "2026-07-02T19:02:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T19:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T19:02:52Z" name: switch-graph-208dc.18be8e4b6988d32a namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: 6c574c45-4c59-463d-948b-877d5f90768e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:41:51.927109Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a5373-79d8b76794-sl4kq namespace: kserve-ci-e2e-test resourceVersion: "16146" uid: f4fd8493-acbe-4086-b17c-f43e0381c905 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-a5373-79d8b76794-sl4kq to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:41:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:41:51Z" name: switch-graph-a5373-79d8b76794-sl4kq.18be8d2983a545ba namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: a106a583-1496-4b46-97d7-a5058bc92872 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:41:52Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a5373-79d8b76794-sl4kq namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: f4fd8493-acbe-4086-b17c-f43e0381c905 kind: Event lastTimestamp: "2026-07-02T18:41:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-a5373-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T18:41:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:52Z" name: switch-graph-a5373-79d8b76794-sl4kq.18be8d298ed6bb34 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: f2a00e33-2819-4217-9625-ed8b4f4e6ec3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:53Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a5373-79d8b76794-sl4kq namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: f4fd8493-acbe-4086-b17c-f43e0381c905 kind: Event lastTimestamp: "2026-07-02T18:41:53Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:41:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:41:53Z" name: switch-graph-a5373-79d8b76794-sl4kq.18be8d29f767bbe3 namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: e52c8e48-dd23-469a-b047-29d044a8ceea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a5373} kind: Pod name: switch-graph-a5373-79d8b76794-sl4kq namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: f4fd8493-acbe-4086-b17c-f43e0381c905 kind: Event lastTimestamp: "2026-07-02T18:41:53Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:75c3c56106a5b8ed3fd4cb951cdacfbe184ab3e990f6497776c0954efdfa10b2" already present on machine metadata: creationTimestamp: "2026-07-02T18:41:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:53Z" name: switch-graph-a5373-79d8b76794-sl4kq.18be8d29f8aceb38 namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: acf7d533-e429-459f-a82f-350ae193b2cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a5373} kind: Pod name: switch-graph-a5373-79d8b76794-sl4kq namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: f4fd8493-acbe-4086-b17c-f43e0381c905 kind: Event lastTimestamp: "2026-07-02T18:41:53Z" message: 'Created container: switch-graph-a5373' metadata: creationTimestamp: "2026-07-02T18:41:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:53Z" name: switch-graph-a5373-79d8b76794-sl4kq.18be8d29fb7fb3b7 namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: fb3f843b-85e8-4b1b-aa45-4166e0193ee8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a5373} kind: Pod name: switch-graph-a5373-79d8b76794-sl4kq namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: f4fd8493-acbe-4086-b17c-f43e0381c905 kind: Event lastTimestamp: "2026-07-02T18:41:53Z" message: Started container switch-graph-a5373 metadata: creationTimestamp: "2026-07-02T18:41:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:53Z" name: switch-graph-a5373-79d8b76794-sl4kq.18be8d29fbe7d9d7 namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: 497a6d39-ac6a-4f2d-9857-87da6f533766 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:50:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a5373} kind: Pod name: switch-graph-a5373-79d8b76794-sl4kq namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: f4fd8493-acbe-4086-b17c-f43e0381c905 kind: Event lastTimestamp: "2026-07-02T18:50:16Z" message: Stopping container switch-graph-a5373 metadata: creationTimestamp: "2026-07-02T18:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:16Z" name: switch-graph-a5373-79d8b76794-sl4kq.18be8d9f0acf68a0 namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: 9c34fd49-8d93-4767-8663-3c5830ab6b6c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a5373} kind: Pod name: switch-graph-a5373-79d8b76794-sl4kq namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: f4fd8493-acbe-4086-b17c-f43e0381c905 kind: Event lastTimestamp: "2026-07-02T18:50:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T18:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:50:45Z" name: switch-graph-a5373-79d8b76794-sl4kq.18be8d9febae77d5 namespace: kserve-ci-e2e-test resourceVersion: "20080" uid: 88b63c7a-24fb-44dc-a3a4-da648b5e4f9c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-a5373-79d8b76794 namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: 2cfa60f5-971c-4de0-8167-20f3a1122ec0 kind: Event lastTimestamp: "2026-07-02T18:41:51Z" message: 'Created pod: switch-graph-a5373-79d8b76794-sl4kq' metadata: creationTimestamp: "2026-07-02T18:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:51Z" name: switch-graph-a5373-79d8b76794.18be8d29833ce942 namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 6a738b79-3dcf-4b0f-9022-966c3c50fa43 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-a5373 namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: d64bfbc4-f49f-4992-bac3-39698ab3db1d kind: Event lastTimestamp: "2026-07-02T18:41:51Z" message: Scaled up replica set switch-graph-a5373-79d8b76794 from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:41:51Z" name: switch-graph-a5373.18be8d2982a23337 namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: 7423bd70-b6cc-427f-82d2-4329fb0fa5ce reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:41:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-a5373 namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 10f66e58-33f0-42aa-ab34-f71e0b4bd33d kind: Event lastTimestamp: "2026-07-02T18:41:51Z" message: InferenceGraph [switch-graph-a5373] is Ready metadata: creationTimestamp: "2026-07-02T18:41:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:41:51Z" name: switch-graph-a5373.18be8d29878a4f87 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 1b0fead5-b970-4997-ae94-a1157f165b4b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:42:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a5373 namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 5a37a5ac-f315-4c86-b3ef-43d6adb10230 kind: Event lastTimestamp: "2026-07-02T18:42:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:21Z" name: switch-graph-a5373.18be8d2d0123bfb1 namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 224eb045-f759-483a-b4c1-fbb2533668f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T18:42:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a5373 namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 5a37a5ac-f315-4c86-b3ef-43d6adb10230 kind: Event lastTimestamp: "2026-07-02T18:42:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T18:42:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:42:21Z" name: switch-graph-a5373.18be8d2d015dba1e namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 18711c33-5b54-4903-846e-ffad617cd29f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T18:40:41.588812Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-bb242-85cccb76bd-59zpf namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: f5e2945e-e7d0-4c90-8f2c-0d0e46537827 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-bb242-85cccb76bd-59zpf to ip-10-0-130-23.ec2.internal metadata: creationTimestamp: "2026-07-02T18:40:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T18:40:41Z" name: switch-graph-bb242-85cccb76bd-59zpf.18be8d19232818a3 namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: c9cfccb5-ed82-4ef3-9400-999d8c23f563 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5f4458674c-wmhfk source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:42Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-bb242-85cccb76bd-59zpf namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: f5e2945e-e7d0-4c90-8f2c-0d0e46537827 kind: Event lastTimestamp: "2026-07-02T18:40:42Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T18:40:42Z" name: switch-graph-bb242-85cccb76bd-59zpf.18be8d193cba0240 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: a96aee16-b558-40f0-9b51-3fc98e5138f6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bb242} kind: Pod name: switch-graph-bb242-85cccb76bd-59zpf namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: f5e2945e-e7d0-4c90-8f2c-0d0e46537827 kind: Event lastTimestamp: "2026-07-02T18:40:42Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:75c3c56106a5b8ed3fd4cb951cdacfbe184ab3e990f6497776c0954efdfa10b2" metadata: creationTimestamp: "2026-07-02T18:40:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:42Z" name: switch-graph-bb242-85cccb76bd-59zpf.18be8d193df5b747 namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: 97a034a0-c6bc-4872-b42d-0f43c58392f2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bb242} kind: Pod name: switch-graph-bb242-85cccb76bd-59zpf namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: f5e2945e-e7d0-4c90-8f2c-0d0e46537827 kind: Event lastTimestamp: "2026-07-02T18:40:44Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:75c3c56106a5b8ed3fd4cb951cdacfbe184ab3e990f6497776c0954efdfa10b2" in 2.502s (2.502s including waiting). Image size: 218072665 bytes.' metadata: creationTimestamp: "2026-07-02T18:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:44Z" name: switch-graph-bb242-85cccb76bd-59zpf.18be8d19d31cbbb9 namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: 85b4bccd-716d-4d8f-8eeb-6eb79a3d3766 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bb242} kind: Pod name: switch-graph-bb242-85cccb76bd-59zpf namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: f5e2945e-e7d0-4c90-8f2c-0d0e46537827 kind: Event lastTimestamp: "2026-07-02T18:40:44Z" message: 'Created container: switch-graph-bb242' metadata: creationTimestamp: "2026-07-02T18:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:44Z" name: switch-graph-bb242-85cccb76bd-59zpf.18be8d19d6c29975 namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: 8b5a5d05-c0f7-46d6-a373-306cbc12b9d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bb242} kind: Pod name: switch-graph-bb242-85cccb76bd-59zpf namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: f5e2945e-e7d0-4c90-8f2c-0d0e46537827 kind: Event lastTimestamp: "2026-07-02T18:40:44Z" message: Started container switch-graph-bb242 metadata: creationTimestamp: "2026-07-02T18:40:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:44Z" name: switch-graph-bb242-85cccb76bd-59zpf.18be8d19d72443cd namespace: kserve-ci-e2e-test resourceVersion: "15200" uid: e4e461eb-554b-45ed-b6c5-31b667ec912e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bb242} kind: Pod name: switch-graph-bb242-85cccb76bd-59zpf namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: f5e2945e-e7d0-4c90-8f2c-0d0e46537827 kind: Event lastTimestamp: "2026-07-02T18:40:51Z" message: Stopping container switch-graph-bb242 metadata: creationTimestamp: "2026-07-02T18:40:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:40:51Z" name: switch-graph-bb242-85cccb76bd-59zpf.18be8d1b7fcaa434 namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: ed4385d0-1d4d-44d6-bc72-5aaf88bb2c27 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T18:40:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-bb242} kind: Pod name: switch-graph-bb242-85cccb76bd-59zpf namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: f5e2945e-e7d0-4c90-8f2c-0d0e46537827 kind: Event lastTimestamp: "2026-07-02T18:41:21Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T18:40:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T18:41:21Z" name: switch-graph-bb242-85cccb76bd-59zpf.18be8d1c85da9d2e namespace: kserve-ci-e2e-test resourceVersion: "15666" uid: cf996842-e2fa-45c8-b609-d7da828f0871 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-23.ec2.internal source: component: kubelet host: ip-10-0-130-23.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-bb242-85cccb76bd namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: 40060c8c-905b-4fb0-9a09-e7adfe5c620c kind: Event lastTimestamp: "2026-07-02T18:40:41Z" message: 'Created pod: switch-graph-bb242-85cccb76bd-59zpf' metadata: creationTimestamp: "2026-07-02T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:41Z" name: switch-graph-bb242-85cccb76bd.18be8d1922b46089 namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 507d49dd-353d-4207-91fb-ece367f208da reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-bb242 namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: c80f8f9a-f7f5-4b01-afd7-0606944b4647 kind: Event lastTimestamp: "2026-07-02T18:40:41Z" message: Scaled up replica set switch-graph-bb242-85cccb76bd from 0 to 1 metadata: creationTimestamp: "2026-07-02T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T18:40:41Z" name: switch-graph-bb242.18be8d1921ed72d4 namespace: kserve-ci-e2e-test resourceVersion: "15138" uid: 2ef15623-14fa-4a91-80ea-1fc585980d66 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-bb242 namespace: kserve-ci-e2e-test resourceVersion: "15156" uid: cd4352eb-a511-4527-b7f9-224d3580c1eb kind: Event lastTimestamp: "2026-07-02T18:40:41Z" message: InferenceGraph [switch-graph-bb242] is Ready metadata: creationTimestamp: "2026-07-02T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:40:41Z" name: switch-graph-bb242.18be8d1927e71868 namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 2e9e9ac5-f49c-4c4b-8bab-b0bc114d3962 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-bb242 namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: cd4352eb-a511-4527-b7f9-224d3580c1eb kind: Event lastTimestamp: "2026-07-02T18:40:41Z" message: 'Failed to update status for InferenceGraph "switch-graph-bb242": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-bb242": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:40:41Z" name: switch-graph-bb242.18be8d192a761c1d namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 2038b2f2-e4d8-428c-8da8-24063167d7ad reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T18:40:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-bb242 namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: cd4352eb-a511-4527-b7f9-224d3580c1eb kind: Event lastTimestamp: "2026-07-02T18:40:41Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-bb242": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T18:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T18:40:41Z" name: switch-graph-bb242.18be8d192a766eee namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: 2d6d4807-872f-4682-b8b1-e8ea6d6035ba reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning kind: EventList metadata: resourceVersion: "33735"