--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:10:00.499563Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-27911-77bb65458-2c7ns namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: cc0f9c39-cd1a-4645-b44f-03fa6432d7c1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-27911-77bb65458-2c7ns to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:10:00Z" name: ensemble-graph-27911-77bb65458-2c7ns.18d5e2cdcddc1ced namespace: kserve-ci-e2e-test resourceVersion: "21890" uid: 2f39344d-0a2c-4b9d-874b-c0ac56fe93c1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-27911-77bb65458-2c7ns namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: cc0f9c39-cd1a-4645-b44f-03fa6432d7c1 kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-27911-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:00Z" name: ensemble-graph-27911-77bb65458-2c7ns.18d5e2cdd9df093d namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 6e600f5f-d019-4ac8-b8d3-29e36f5da7ae reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-27911-77bb65458-2c7ns namespace: kserve-ci-e2e-test resourceVersion: "21894" uid: cc0f9c39-cd1a-4645-b44f-03fa6432d7c1 kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:10:01Z" name: ensemble-graph-27911-77bb65458-2c7ns.18d5e2ce13ac546f namespace: kserve-ci-e2e-test resourceVersion: "21937" uid: 523efb67-87d9-4386-a00b-acd5856822c7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-27911} kind: Pod name: ensemble-graph-27911-77bb65458-2c7ns namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: cc0f9c39-cd1a-4645-b44f-03fa6432d7c1 kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:83a04242aef3bc251b4da1ba3ce544c38d7df9880cbb933157aae8432a36f471" already present on machine metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: ensemble-graph-27911-77bb65458-2c7ns.18d5e2ce14e3e3cf namespace: kserve-ci-e2e-test resourceVersion: "21939" uid: ecd52464-2c1d-4c11-853d-8026837901ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-27911} kind: Pod name: ensemble-graph-27911-77bb65458-2c7ns namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: cc0f9c39-cd1a-4645-b44f-03fa6432d7c1 kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: 'Created container: ensemble-graph-27911' metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: ensemble-graph-27911-77bb65458-2c7ns.18d5e2ce1783feb4 namespace: kserve-ci-e2e-test resourceVersion: "21940" uid: 38298df8-eb34-461d-9b07-c5f0dfb6b972 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-27911} kind: Pod name: ensemble-graph-27911-77bb65458-2c7ns namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: cc0f9c39-cd1a-4645-b44f-03fa6432d7c1 kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: Started container ensemble-graph-27911 metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: ensemble-graph-27911-77bb65458-2c7ns.18d5e2ce1804086f namespace: kserve-ci-e2e-test resourceVersion: "21941" uid: d522d0fc-a003-4536-b38c-a50b253a510b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-27911} kind: Pod name: ensemble-graph-27911-77bb65458-2c7ns namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: cc0f9c39-cd1a-4645-b44f-03fa6432d7c1 kind: Event lastTimestamp: "2026-09-16T19:18:15Z" message: Stopping container ensemble-graph-27911 metadata: creationTimestamp: "2026-09-16T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:15Z" name: ensemble-graph-27911-77bb65458-2c7ns.18d5e340fc03eb03 namespace: kserve-ci-e2e-test resourceVersion: "25739" uid: 5174a21b-da09-40d6-9ab7-4b87f93a5a48 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:18:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-27911} kind: Pod name: ensemble-graph-27911-77bb65458-2c7ns namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: cc0f9c39-cd1a-4645-b44f-03fa6432d7c1 kind: Event lastTimestamp: "2026-09-16T19:18:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:43Z" name: ensemble-graph-27911-77bb65458-2c7ns.18d5e341a99b3afb namespace: kserve-ci-e2e-test resourceVersion: "26085" uid: 45b6f9f9-e41b-4ee6-97e5-72b7b99488cc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-27911-77bb65458 namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: 1325dcfb-caa7-4d21-86be-3c273cc8fcd3 kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: 'Created pod: ensemble-graph-27911-77bb65458-2c7ns' metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:00Z" name: ensemble-graph-27911-77bb65458.18d5e2cdcd62ccc0 namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: 6fdd4e50-b6b0-46c8-81d2-0bb4255ca292 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-27911 namespace: kserve-ci-e2e-test resourceVersion: "21878" uid: b45d009e-b258-4c7e-bc7f-90f582e31da0 kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: Scaled up replica set ensemble-graph-27911-77bb65458 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:00Z" name: ensemble-graph-27911.18d5e2cdcc980609 namespace: kserve-ci-e2e-test resourceVersion: "21883" uid: 9fbf897c-1940-4136-9d2a-7f80c2bb691c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-27911 namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: aaa9d18d-1e09-48a1-b1ff-4218880b078c kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: InferenceGraph [ensemble-graph-27911] is Ready metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:10:00Z" name: ensemble-graph-27911.18d5e2cdd20d1759 namespace: kserve-ci-e2e-test resourceVersion: "21902" uid: fd0de5ef-0074-4256-ae8b-ab4dd87b8f77 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-27911 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: 31094678-addd-4134-b7e8-82b20031eca7 kind: Event lastTimestamp: "2026-09-16T19:10:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:15Z" name: ensemble-graph-27911.18d5e2d14b3c921d namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: 95af1819-68f7-43de-a738-cb5114b7fce8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-27911 namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: 31094678-addd-4134-b7e8-82b20031eca7 kind: Event lastTimestamp: "2026-09-16T19:10:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:15Z" name: ensemble-graph-27911.18d5e2d14bc2becd namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: 904b3ced-1252-43d8-a693-da3f7b7cfa18 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-27911 namespace: kserve-ci-e2e-test resourceVersion: "22078" uid: 31094678-addd-4134-b7e8-82b20031eca7 kind: Event lastTimestamp: "2026-09-16T19:10:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:10:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:30Z" name: ensemble-graph-27911.18d5e2d4caa29608 namespace: kserve-ci-e2e-test resourceVersion: "22192" uid: 8b2710f7-3256-48d6-b8ef-f44d9b3c5142 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-27911 namespace: kserve-ci-e2e-test resourceVersion: "22078" uid: 31094678-addd-4134-b7e8-82b20031eca7 kind: Event lastTimestamp: "2026-09-16T19:10:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:10:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:30Z" name: ensemble-graph-27911.18d5e2d4caf33b21 namespace: kserve-ci-e2e-test resourceVersion: "22194" uid: 5c733497-6a41-4443-b699-ddcfbc323568 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:08:50.151123Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-443e6-7c548ddd5f-b4hgw namespace: kserve-ci-e2e-test resourceVersion: "20909" uid: d5946f17-a43a-473c-b210-ce01616020bb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-443e6-7c548ddd5f-b4hgw to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:08:50Z" name: ensemble-graph-443e6-7c548ddd5f-b4hgw.18d5e2bd6cc4283d namespace: kserve-ci-e2e-test resourceVersion: "20914" uid: 99513af4-336e-4a7b-b3bc-9d2c9f39ae19 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-443e6-7c548ddd5f-b4hgw namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: d5946f17-a43a-473c-b210-ce01616020bb kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:08:50Z" name: ensemble-graph-443e6-7c548ddd5f-b4hgw.18d5e2bd8e7ed6ad namespace: kserve-ci-e2e-test resourceVersion: "20953" uid: 06b666bb-8a1e-477b-be63-2531e953f91a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-443e6} kind: Pod name: ensemble-graph-443e6-7c548ddd5f-b4hgw namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: d5946f17-a43a-473c-b210-ce01616020bb kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:83a04242aef3bc251b4da1ba3ce544c38d7df9880cbb933157aae8432a36f471" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:50Z" name: ensemble-graph-443e6-7c548ddd5f-b4hgw.18d5e2bd8fce378b namespace: kserve-ci-e2e-test resourceVersion: "20957" uid: 968aad6b-544f-4093-a2fa-a40504961708 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-443e6} kind: Pod name: ensemble-graph-443e6-7c548ddd5f-b4hgw namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: d5946f17-a43a-473c-b210-ce01616020bb kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: 'Created container: ensemble-graph-443e6' metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:50Z" name: ensemble-graph-443e6-7c548ddd5f-b4hgw.18d5e2bd92903d82 namespace: kserve-ci-e2e-test resourceVersion: "20961" uid: 2b37d998-3a80-49c6-9b2e-05b2d6d989df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-443e6} kind: Pod name: ensemble-graph-443e6-7c548ddd5f-b4hgw namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: d5946f17-a43a-473c-b210-ce01616020bb kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: Started container ensemble-graph-443e6 metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:50Z" name: ensemble-graph-443e6-7c548ddd5f-b4hgw.18d5e2bd92f7e1ce namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: 8dfe5d34-b6df-4f92-ba72-416cdda23fa0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-443e6} kind: Pod name: ensemble-graph-443e6-7c548ddd5f-b4hgw namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: d5946f17-a43a-473c-b210-ce01616020bb kind: Event lastTimestamp: "2026-09-16T19:09:00Z" message: Stopping container ensemble-graph-443e6 metadata: creationTimestamp: "2026-09-16T19:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:00Z" name: ensemble-graph-443e6-7c548ddd5f-b4hgw.18d5e2bfc98de5dd namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: 971b2580-391a-4770-9cfe-f1ed6bff8c12 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-443e6} kind: Pod name: ensemble-graph-443e6-7c548ddd5f-b4hgw namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: d5946f17-a43a-473c-b210-ce01616020bb kind: Event lastTimestamp: "2026-09-16T19:09:26Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:26Z" name: ensemble-graph-443e6-7c548ddd5f-b4hgw.18d5e2c024990771 namespace: kserve-ci-e2e-test resourceVersion: "21430" uid: 954a53ba-8647-4b43-ad55-1fc80f71582c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-443e6-7c548ddd5f namespace: kserve-ci-e2e-test resourceVersion: "20906" uid: 0ffbc73b-a650-4828-9bf9-847d7df4ff2c kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: 'Created pod: ensemble-graph-443e6-7c548ddd5f-b4hgw' metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:50Z" name: ensemble-graph-443e6-7c548ddd5f.18d5e2bd6c527e2f namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: 6f6ea0ca-e15c-431b-bbe2-4fe126083453 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-443e6 namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 51449c29-71b2-46c5-b40f-0fb20eb300fa kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: Scaled up replica set ensemble-graph-443e6-7c548ddd5f from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:50Z" name: ensemble-graph-443e6.18d5e2bd6b94ddbf namespace: kserve-ci-e2e-test resourceVersion: "20907" uid: 4ae8c088-1230-4b78-8693-04bba5219cf7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-443e6 namespace: kserve-ci-e2e-test resourceVersion: "20929" uid: d2491894-d287-4903-9472-d04f41ecadbf kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: InferenceGraph [ensemble-graph-443e6] is Ready metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:08:50Z" name: ensemble-graph-443e6.18d5e2bd71e65f5b namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: f61b5125-203e-4bd4-a919-d8a301b3185f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-443e6 namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: d2491894-d287-4903-9472-d04f41ecadbf kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-443e6": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-443e6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:08:50Z" name: ensemble-graph-443e6.18d5e2bd7481357d namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: 6bfa782d-5269-40af-8d42-6736594d68b7 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-443e6 namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: d2491894-d287-4903-9472-d04f41ecadbf kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-443e6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:08:50Z" name: ensemble-graph-443e6.18d5e2bd748199a3 namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: a5ef5bda-13fe-4910-bb7c-7742c2f1a82f reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:19:25.815303Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4 namespace: kserve-ci-e2e-test resourceVersion: "26705" uid: 43badc11-3b5e-4dd5-92b5-1cde83fe1b81 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-17a4d-predictor-6db794b454-pbjn4 to ip-10-0-142-175.ec2.internal metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:19:25Z" name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4.18d5e3516d4dd08a namespace: kserve-ci-e2e-test resourceVersion: "26709" uid: 99708b0c-29ae-4887-aef2-a1960d545f53 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 43badc11-3b5e-4dd5-92b5-1cde83fe1b81 kind: Event lastTimestamp: "2026-09-16T19:19:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-17a4d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:26Z" name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4.18d5e3517a383d6f namespace: kserve-ci-e2e-test resourceVersion: "26725" uid: 4eec7afa-f09d-43ca-9b0f-6be20cd16235 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4 namespace: kserve-ci-e2e-test resourceVersion: "26715" uid: 43badc11-3b5e-4dd5-92b5-1cde83fe1b81 kind: Event lastTimestamp: "2026-09-16T19:19:26Z" message: Add eth0 [10.133.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:19:26Z" name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4.18d5e351b389589d namespace: kserve-ci-e2e-test resourceVersion: "26741" uid: ad2d78ba-bfd9-42e8-b3bd-7bdca1f80759 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 43badc11-3b5e-4dd5-92b5-1cde83fe1b81 kind: Event lastTimestamp: "2026-09-16T19:19:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:27Z" name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4.18d5e351b564830c namespace: kserve-ci-e2e-test resourceVersion: "26745" uid: 770dba5f-f310-4c3d-a800-b9d944588d5f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 43badc11-3b5e-4dd5-92b5-1cde83fe1b81 kind: Event lastTimestamp: "2026-09-16T19:19:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:27Z" name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4.18d5e351b8abfbc6 namespace: kserve-ci-e2e-test resourceVersion: "26746" uid: 6611da63-6e61-4567-a177-507d783aec62 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 43badc11-3b5e-4dd5-92b5-1cde83fe1b81 kind: Event lastTimestamp: "2026-09-16T19:19:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:27Z" name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4.18d5e351b93b590d namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: de0aaf41-ee04-4fba-8c3d-cdb0a919782e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 43badc11-3b5e-4dd5-92b5-1cde83fe1b81 kind: Event lastTimestamp: "2026-09-16T19:19:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:27Z" name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4.18d5e351b947ce2a namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: 51e556f1-710f-4b70-9f90-58b8598ba711 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 43badc11-3b5e-4dd5-92b5-1cde83fe1b81 kind: Event lastTimestamp: "2026-09-16T19:19:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:27Z" name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4.18d5e351bc83e608 namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: d914b695-02ad-49a2-9207-14b9c92c3ebf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 43badc11-3b5e-4dd5-92b5-1cde83fe1b81 kind: Event lastTimestamp: "2026-09-16T19:19:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:27Z" name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4.18d5e351bcfd5a5c namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: 17b25e9e-aea2-4078-a9ea-16785ca4df15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:19:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 43badc11-3b5e-4dd5-92b5-1cde83fe1b81 kind: Event lastTimestamp: "2026-09-16T19:20:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:20:04Z" name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4.18d5e35222ee9d33 namespace: kserve-ci-e2e-test resourceVersion: "27097" uid: 04f29642-40cc-41b1-a682-4010e66855e6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 43badc11-3b5e-4dd5-92b5-1cde83fe1b81 kind: Event lastTimestamp: "2026-09-16T19:28:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:28:40Z" name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4.18d5e3d28d751a2b namespace: kserve-ci-e2e-test resourceVersion: "30807" uid: c9323d53-6906-4271-8f80-e067a0181c87 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 43badc11-3b5e-4dd5-92b5-1cde83fe1b81 kind: Event lastTimestamp: "2026-09-16T19:28:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:28:40Z" name: error-404-isvc-17a4d-predictor-6db794b454-pbjn4.18d5e3d28d763f7f namespace: kserve-ci-e2e-test resourceVersion: "30808" uid: 63e020f9-dcdb-414d-9754-34b7c7ceb7d6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-17a4d-predictor-6db794b454 namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 84f4d571-27f3-477c-b2a0-3e5c8d127acd kind: Event lastTimestamp: "2026-09-16T19:19:25Z" message: 'Created pod: error-404-isvc-17a4d-predictor-6db794b454-pbjn4' metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:25Z" name: error-404-isvc-17a4d-predictor-6db794b454.18d5e3516ce15091 namespace: kserve-ci-e2e-test resourceVersion: "26706" uid: 3b68af83-6aff-4fb8-bacb-368644882d4c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-17a4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26697" uid: defcb181-cf8f-4555-a76b-bad8d84fb96e kind: Event lastTimestamp: "2026-09-16T19:19:25Z" message: Scaled up replica set error-404-isvc-17a4d-predictor-6db794b454 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:25Z" name: error-404-isvc-17a4d-predictor.18d5e3516afccb00 namespace: kserve-ci-e2e-test resourceVersion: "26700" uid: 080f65eb-bdb4-44a9-b483-8971dd2a553b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:19:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-17a4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26698" uid: ece9a061-03fe-48d7-be70-d3045902c9c1 kind: Event lastTimestamp: "2026-09-16T19:19:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:19:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:55Z" name: error-404-isvc-17a4d-predictor.18d5e354e9700139 namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: 05c929ca-6105-4327-aa1b-9c8c57f7fe26 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:19:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-17a4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26698" uid: ece9a061-03fe-48d7-be70-d3045902c9c1 kind: Event lastTimestamp: "2026-09-16T19:19:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:19:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:55Z" name: error-404-isvc-17a4d-predictor.18d5e354e9b1e9de namespace: kserve-ci-e2e-test resourceVersion: "27012" uid: 112b8154-1179-4438-b425-aa215c5feeca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:20:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-17a4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26880" uid: ece9a061-03fe-48d7-be70-d3045902c9c1 kind: Event lastTimestamp: "2026-09-16T19:20:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:20:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:25Z" name: error-404-isvc-17a4d-predictor.18d5e35be7435f92 namespace: kserve-ci-e2e-test resourceVersion: "27336" uid: eb39e0b1-0a55-4145-89c5-4387be7bbca2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:20:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-17a4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26880" uid: ece9a061-03fe-48d7-be70-d3045902c9c1 kind: Event lastTimestamp: "2026-09-16T19:20:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:20:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:25Z" name: error-404-isvc-17a4d-predictor.18d5e35be7b93f2d namespace: kserve-ci-e2e-test resourceVersion: "27338" uid: 54213b0e-b8ab-4f20-bbf6-20d104005724 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:19:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-17a4d namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: e0e937a7-59aa-47d4-abf8-292c8eb6daaa kind: Event lastTimestamp: "2026-09-16T19:20:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:20:15Z" name: error-404-isvc-17a4d.18d5e3516af5c4a5 namespace: kserve-ci-e2e-test resourceVersion: "27194" uid: fb97e9a1-de74-42bc-9e07-c2131332444a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-17a4d namespace: kserve-ci-e2e-test resourceVersion: "27191" uid: e0e937a7-59aa-47d4-abf8-292c8eb6daaa kind: Event lastTimestamp: "2026-09-16T19:20:14Z" message: InferenceService [error-404-isvc-17a4d] is Ready metadata: creationTimestamp: "2026-09-16T19:20:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:20:14Z" name: error-404-isvc-17a4d.18d5e35ce08b5496 namespace: kserve-ci-e2e-test resourceVersion: "27192" uid: b0bffa1f-ef6a-443d-8e02-7de928d675ce reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:09:00.771835Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-27911-predictor-6965d7c55b-x57m9 namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: d8777970-6d1f-4eac-b74e-8a740e4e0aa4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-27911-predictor-6965d7c55b-x57m9 to ip-10-0-142-175.ec2.internal metadata: creationTimestamp: "2026-09-16T19:09:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:09:00Z" name: error-404-isvc-27911-predictor-6965d7c55b-x57m9.18d5e2bfe5cf6c46 namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: e775a791-a2bf-4247-8ba8-c8bd563411fb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-27911-predictor-6965d7c55b-x57m9 namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: d8777970-6d1f-4eac-b74e-8a740e4e0aa4 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:09:01Z" name: error-404-isvc-27911-predictor-6965d7c55b-x57m9.18d5e2c0081d6563 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: 6e131960-0ba5-4292-83bf-bcc6ff455c4c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-27911-predictor-6965d7c55b-x57m9 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: d8777970-6d1f-4eac-b74e-8a740e4e0aa4 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:01Z" name: error-404-isvc-27911-predictor-6965d7c55b-x57m9.18d5e2c009dfff10 namespace: kserve-ci-e2e-test resourceVersion: "21196" uid: 45204620-616e-43a0-9ebe-75c0b8882dc5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-27911-predictor-6965d7c55b-x57m9 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: d8777970-6d1f-4eac-b74e-8a740e4e0aa4 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:01Z" name: error-404-isvc-27911-predictor-6965d7c55b-x57m9.18d5e2c00cea84d7 namespace: kserve-ci-e2e-test resourceVersion: "21197" uid: 1ff946ee-65b6-4813-8dda-2f1f3fcc2cea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-27911-predictor-6965d7c55b-x57m9 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: d8777970-6d1f-4eac-b74e-8a740e4e0aa4 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:01Z" name: error-404-isvc-27911-predictor-6965d7c55b-x57m9.18d5e2c00d6c812f namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: 282b25dc-6f2c-49a1-a09d-4221a07d5596 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-27911-predictor-6965d7c55b-x57m9 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: d8777970-6d1f-4eac-b74e-8a740e4e0aa4 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:01Z" name: error-404-isvc-27911-predictor-6965d7c55b-x57m9.18d5e2c00d793f60 namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 0ed9ac8c-b128-48bc-9f72-12cc6efe6d1b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-27911-predictor-6965d7c55b-x57m9 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: d8777970-6d1f-4eac-b74e-8a740e4e0aa4 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:01Z" name: error-404-isvc-27911-predictor-6965d7c55b-x57m9.18d5e2c010e1b396 namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: 1dee8509-93c0-4a40-adbd-6e158099f526 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-27911-predictor-6965d7c55b-x57m9 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: d8777970-6d1f-4eac-b74e-8a740e4e0aa4 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:01Z" name: error-404-isvc-27911-predictor-6965d7c55b-x57m9.18d5e2c011663a74 namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: ae087c2b-0657-46cb-a21b-90d3e4b9f094 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T19:09:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-27911-predictor-6965d7c55b-x57m9 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: d8777970-6d1f-4eac-b74e-8a740e4e0aa4 kind: Event lastTimestamp: "2026-09-16T19:18:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:18Z" name: error-404-isvc-27911-predictor-6965d7c55b-x57m9.18d5e2c05c815094 namespace: kserve-ci-e2e-test resourceVersion: "25888" uid: 45bb4da6-5661-4ec6-8985-ee72fbea2cb2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-27911-predictor-6965d7c55b-x57m9 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: d8777970-6d1f-4eac-b74e-8a740e4e0aa4 kind: Event lastTimestamp: "2026-09-16T19:18:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:15Z" name: error-404-isvc-27911-predictor-6965d7c55b-x57m9.18d5e3410f2ad4c0 namespace: kserve-ci-e2e-test resourceVersion: "25802" uid: 8b26c2f4-b648-49ba-9f39-71485470ce71 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-27911-predictor-6965d7c55b-x57m9 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: d8777970-6d1f-4eac-b74e-8a740e4e0aa4 kind: Event lastTimestamp: "2026-09-16T19:18:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:15Z" name: error-404-isvc-27911-predictor-6965d7c55b-x57m9.18d5e3410f2b6598 namespace: kserve-ci-e2e-test resourceVersion: "25805" uid: d0c3176a-ae93-4a49-a8b4-a3eeb54fa284 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-27911-predictor-6965d7c55b-x57m9 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: d8777970-6d1f-4eac-b74e-8a740e4e0aa4 kind: Event lastTimestamp: "2026-09-16T19:18:18Z" message: 'Readiness probe failed: Get "https://10.133.0.42:8643/healthz": dial tcp 10.133.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:18:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:18Z" name: error-404-isvc-27911-predictor-6965d7c55b-x57m9.18d5e341d0f386ac namespace: kserve-ci-e2e-test resourceVersion: "25887" uid: 9284d0ed-ecaa-4382-88a1-14d3e0d12dc9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-27911-predictor-6965d7c55b namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: 5c10821b-037d-4e7f-82c7-0931b60aca0b kind: Event lastTimestamp: "2026-09-16T19:09:00Z" message: 'Created pod: error-404-isvc-27911-predictor-6965d7c55b-x57m9' metadata: creationTimestamp: "2026-09-16T19:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:00Z" name: error-404-isvc-27911-predictor-6965d7c55b.18d5e2bfe4995b1c namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: a2fc597d-5d76-4e5b-9784-ad241c05a34e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-27911-predictor namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 4234f7fe-5251-4faf-9e3f-cb8fdaec9f7a kind: Event lastTimestamp: "2026-09-16T19:09:00Z" message: Scaled up replica set error-404-isvc-27911-predictor-6965d7c55b from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:00Z" name: error-404-isvc-27911-predictor.18d5e2bfe12bcb10 namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: 341d7b4b-89ad-4d2c-9880-ff0961211e03 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:09:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-27911-predictor namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: 97ef8d9c-65f2-4e1f-ad11-0a9899e624d2 kind: Event lastTimestamp: "2026-09-16T19:09:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:09:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:30Z" name: error-404-isvc-27911-predictor.18d5e2c35f9433c9 namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: e009d6e0-d414-4527-b6c7-6f518811e763 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:09:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-27911-predictor namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: 97ef8d9c-65f2-4e1f-ad11-0a9899e624d2 kind: Event lastTimestamp: "2026-09-16T19:09:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:09:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:30Z" name: error-404-isvc-27911-predictor.18d5e2c3602880d5 namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: 2121995f-7397-4c63-90de-24db34d5bf72 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:09:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-27911-predictor namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: 97ef8d9c-65f2-4e1f-ad11-0a9899e624d2 kind: Event lastTimestamp: "2026-09-16T19:10:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:09:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:15Z" name: error-404-isvc-27911-predictor.18d5e2ca5e088663 namespace: kserve-ci-e2e-test resourceVersion: "22084" uid: d5b3ccf2-8a8b-4897-baf3-365a3c8efb38 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:09:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-27911-predictor namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: 97ef8d9c-65f2-4e1f-ad11-0a9899e624d2 kind: Event lastTimestamp: "2026-09-16T19:10:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:09:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:15Z" name: error-404-isvc-27911-predictor.18d5e2ca5e812bec namespace: kserve-ci-e2e-test resourceVersion: "22085" uid: 74cf308a-52c1-444a-9dee-68198ff220e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:09:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-27911 namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: a89b2530-50d5-4f82-91cb-bf74f0efa5b9 kind: Event lastTimestamp: "2026-09-16T19:09:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:09:48Z" name: error-404-isvc-27911.18d5e2bfe11acaee namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: 7a106f7e-197a-428e-a9d9-cf37f0ed4edd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-27911 namespace: kserve-ci-e2e-test resourceVersion: "21637" uid: a89b2530-50d5-4f82-91cb-bf74f0efa5b9 kind: Event lastTimestamp: "2026-09-16T19:09:48Z" message: InferenceService [error-404-isvc-27911] is Ready metadata: creationTimestamp: "2026-09-16T19:09:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:09:48Z" name: error-404-isvc-27911.18d5e2cb182ee513 namespace: kserve-ci-e2e-test resourceVersion: "21638" uid: 699b5a06-7896-4789-9655-31ae95049b17 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:18:15.684473Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g namespace: kserve-ci-e2e-test resourceVersion: "25821" uid: bfdbc5b8-e91e-42fe-b995-3b89ce0638b1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g to ip-10-0-142-175.ec2.internal metadata: creationTimestamp: "2026-09-16T19:18:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:18:15Z" name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g.18d5e341192e4d7d namespace: kserve-ci-e2e-test resourceVersion: "25825" uid: 27c89ef6-01a8-4e6f-bb96-7b671c44df12 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g namespace: kserve-ci-e2e-test resourceVersion: "25831" uid: bfdbc5b8-e91e-42fe-b995-3b89ce0638b1 kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:18:16Z" name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g.18d5e3413b3d2a4c namespace: kserve-ci-e2e-test resourceVersion: "25849" uid: 452abbe3-8bd6-487f-8e45-141814be03d3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: bfdbc5b8-e91e-42fe-b995-3b89ce0638b1 kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:16Z" name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g.18d5e3413cc06069 namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: 848b5975-0799-4ea4-a7c6-bf6f98adfbf5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: bfdbc5b8-e91e-42fe-b995-3b89ce0638b1 kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:16Z" name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g.18d5e341400f5376 namespace: kserve-ci-e2e-test resourceVersion: "25852" uid: 1f7001f0-a0d7-4968-96f7-408a411cbf53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: bfdbc5b8-e91e-42fe-b995-3b89ce0638b1 kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:16Z" name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g.18d5e341408dd25e namespace: kserve-ci-e2e-test resourceVersion: "25853" uid: 8812366d-c070-4a00-8c5f-3ac85e3854ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: bfdbc5b8-e91e-42fe-b995-3b89ce0638b1 kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:16Z" name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g.18d5e341409877b8 namespace: kserve-ci-e2e-test resourceVersion: "25855" uid: 0eccbbb0-3137-4811-854b-e87fba668fd5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: bfdbc5b8-e91e-42fe-b995-3b89ce0638b1 kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:16Z" name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g.18d5e3414463a180 namespace: kserve-ci-e2e-test resourceVersion: "25857" uid: 17a0789f-e66d-495a-a0b1-5e30b47bd0bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: bfdbc5b8-e91e-42fe-b995-3b89ce0638b1 kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:16Z" name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g.18d5e34144dae1e7 namespace: kserve-ci-e2e-test resourceVersion: "25858" uid: 39d4124b-e41d-4ad0-8437-061da6a29c87 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:18:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: bfdbc5b8-e91e-42fe-b995-3b89ce0638b1 kind: Event lastTimestamp: "2026-09-16T19:18:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:53Z" name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g.18d5e34184fa1f04 namespace: kserve-ci-e2e-test resourceVersion: "26168" uid: 6e61bf42-7446-4488-9704-d4de4ab7651c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: bfdbc5b8-e91e-42fe-b995-3b89ce0638b1 kind: Event lastTimestamp: "2026-09-16T19:19:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:25Z" name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g.18d5e351624a23ff namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: 683001cb-bed3-44e4-820a-bc096f8c6107 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: bfdbc5b8-e91e-42fe-b995-3b89ce0638b1 kind: Event lastTimestamp: "2026-09-16T19:19:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:25Z" name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g.18d5e351624b3b35 namespace: kserve-ci-e2e-test resourceVersion: "26667" uid: 08935ab5-e3e3-41ac-83a1-f83e8c9ab7f6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g namespace: kserve-ci-e2e-test resourceVersion: "25823" uid: bfdbc5b8-e91e-42fe-b995-3b89ce0638b1 kind: Event lastTimestamp: "2026-09-16T19:19:28Z" message: 'Readiness probe failed: Get "https://10.133.0.44:8643/healthz": dial tcp 10.133.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:28Z" name: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g.18d5e3520d2b06a8 namespace: kserve-ci-e2e-test resourceVersion: "26769" uid: 7d8b7ac6-bcee-416c-81b8-1d3850857471 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3bba6-predictor-67dcb48bc6 namespace: kserve-ci-e2e-test resourceVersion: "25815" uid: b41ce69f-36d1-488f-8480-3d0fb1afbdc7 kind: Event lastTimestamp: "2026-09-16T19:18:15Z" message: 'Created pod: error-404-isvc-3bba6-predictor-67dcb48bc6-8lb5g' metadata: creationTimestamp: "2026-09-16T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:15Z" name: error-404-isvc-3bba6-predictor-67dcb48bc6.18d5e34118bf9413 namespace: kserve-ci-e2e-test resourceVersion: "25822" uid: 9c8f9873-6129-4ac8-8e91-a0ce7d3868f4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3bba6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25813" uid: f4ba5482-af4e-4aab-be8c-f720a1dc7f8b kind: Event lastTimestamp: "2026-09-16T19:18:15Z" message: Scaled up replica set error-404-isvc-3bba6-predictor-67dcb48bc6 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:15Z" name: error-404-isvc-3bba6-predictor.18d5e34115d89b78 namespace: kserve-ci-e2e-test resourceVersion: "25818" uid: 72e7b010-30c6-48f0-bb01-dd25cb4b3c0c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:18:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3bba6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25814" uid: f79a3837-0cff-4f1f-b221-89dd4905d517 kind: Event lastTimestamp: "2026-09-16T19:18:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:18:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:45Z" name: error-404-isvc-3bba6-predictor.18d5e3449434c0e4 namespace: kserve-ci-e2e-test resourceVersion: "26107" uid: 699528dc-f19f-452e-bc5d-0dba98956d50 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:18:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3bba6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25814" uid: f79a3837-0cff-4f1f-b221-89dd4905d517 kind: Event lastTimestamp: "2026-09-16T19:18:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:18:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:45Z" name: error-404-isvc-3bba6-predictor.18d5e3449491aa86 namespace: kserve-ci-e2e-test resourceVersion: "26108" uid: c663def7-e625-46e7-8eef-9dfea9baffc0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:19:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3bba6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25988" uid: f79a3837-0cff-4f1f-b221-89dd4905d517 kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:19:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:15Z" name: error-404-isvc-3bba6-predictor.18d5e34b925498a3 namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: ebf0e900-ea59-406b-85d0-013db01b3b90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:19:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3bba6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25988" uid: f79a3837-0cff-4f1f-b221-89dd4905d517 kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:19:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:15Z" name: error-404-isvc-3bba6-predictor.18d5e34b92c4f037 namespace: kserve-ci-e2e-test resourceVersion: "26489" uid: d747b9ba-dde6-42a6-8c90-94188aaa4113 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:18:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3bba6 namespace: kserve-ci-e2e-test resourceVersion: "25797" uid: af8c7258-fa21-413e-a2d0-1aaf397dcb3e kind: Event lastTimestamp: "2026-09-16T19:19:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:19:03Z" name: error-404-isvc-3bba6.18d5e34115e2e70a namespace: kserve-ci-e2e-test resourceVersion: "26251" uid: 3552cabb-39bc-43f1-a5ef-ffe5a501b704 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3bba6 namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: af8c7258-fa21-413e-a2d0-1aaf397dcb3e kind: Event lastTimestamp: "2026-09-16T19:19:03Z" message: InferenceService [error-404-isvc-3bba6] is Ready metadata: creationTimestamp: "2026-09-16T19:19:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:19:03Z" name: error-404-isvc-3bba6.18d5e34c40920af8 namespace: kserve-ci-e2e-test resourceVersion: "26249" uid: 63a1fdda-d58f-4c8e-8ec5-d4dad09b1b3f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:07:50.342717Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7 namespace: kserve-ci-e2e-test resourceVersion: "20291" uid: 2ee2ad02-86fe-4710-85b1-bf7131f47883 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7 to ip-10-0-142-175.ec2.internal metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:07:50Z" name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7.18d5e2af7fe927c7 namespace: kserve-ci-e2e-test resourceVersion: "20294" uid: e39fe293-2f8d-4239-a391-9b678c69a7bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7 namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 2ee2ad02-86fe-4710-85b1-bf7131f47883 kind: Event lastTimestamp: "2026-09-16T19:07:50Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:07:50Z" name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7.18d5e2afa2066d88 namespace: kserve-ci-e2e-test resourceVersion: "20316" uid: 2891abc3-623f-46f9-93cd-ab64ee2fb30e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 2ee2ad02-86fe-4710-85b1-bf7131f47883 kind: Event lastTimestamp: "2026-09-16T19:07:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:50Z" name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7.18d5e2afa39085b1 namespace: kserve-ci-e2e-test resourceVersion: "20318" uid: c946aaa3-78c1-46fe-8cb5-2d2b3a8cabb6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 2ee2ad02-86fe-4710-85b1-bf7131f47883 kind: Event lastTimestamp: "2026-09-16T19:07:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:51Z" name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7.18d5e2afa7fc179d namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: 4a64151d-376a-44a2-99d5-1d06f5892b5e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 2ee2ad02-86fe-4710-85b1-bf7131f47883 kind: Event lastTimestamp: "2026-09-16T19:07:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:51Z" name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7.18d5e2afa87716c8 namespace: kserve-ci-e2e-test resourceVersion: "20320" uid: 0c195ee1-7d60-4f65-9705-15ccecac35ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 2ee2ad02-86fe-4710-85b1-bf7131f47883 kind: Event lastTimestamp: "2026-09-16T19:07:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:51Z" name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7.18d5e2afa882a828 namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: ae7ecb25-44e0-43fe-b648-ac073823999d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 2ee2ad02-86fe-4710-85b1-bf7131f47883 kind: Event lastTimestamp: "2026-09-16T19:07:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:51Z" name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7.18d5e2afab3ae97d namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: eed53889-38e8-4cbe-8fda-1669afb9a194 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 2ee2ad02-86fe-4710-85b1-bf7131f47883 kind: Event lastTimestamp: "2026-09-16T19:07:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:51Z" name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7.18d5e2afabbd2ca6 namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: 3955ddd9-a3e4-43b6-86fd-82af4be61b60 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 2ee2ad02-86fe-4710-85b1-bf7131f47883 kind: Event lastTimestamp: "2026-09-16T19:08:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:27Z" name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7.18d5e2afbf66b166 namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: e12e9356-ac30-42d0-b533-6f7d87e4abd9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 2ee2ad02-86fe-4710-85b1-bf7131f47883 kind: Event lastTimestamp: "2026-09-16T19:09:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:00Z" name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7.18d5e2bfcf5e086c namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 31459f7e-61c8-4af6-984d-2833f5d62b02 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 2ee2ad02-86fe-4710-85b1-bf7131f47883 kind: Event lastTimestamp: "2026-09-16T19:09:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:00Z" name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7.18d5e2bfcf5f8f0b namespace: kserve-ci-e2e-test resourceVersion: "21110" uid: 0edf6531-17c1-4c6d-b6d8-d764f22a2b0f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: 2ee2ad02-86fe-4710-85b1-bf7131f47883 kind: Event lastTimestamp: "2026-09-16T19:09:02Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:09:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:02Z" name: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7.18d5e2c0479d2e5a namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 28ff2635-c9a2-415c-a56e-de178379f563 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-443e6-predictor-569c8dcb6d namespace: kserve-ci-e2e-test resourceVersion: "20284" uid: 674db0c4-82af-49c1-baa6-480e63579a66 kind: Event lastTimestamp: "2026-09-16T19:07:50Z" message: 'Created pod: error-404-isvc-443e6-predictor-569c8dcb6d-4kpx7' metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:50Z" name: error-404-isvc-443e6-predictor-569c8dcb6d.18d5e2af7ff5875a namespace: kserve-ci-e2e-test resourceVersion: "20295" uid: b338f883-511f-469f-af45-586742c13cde reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-443e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20282" uid: de8f4633-c4fa-4acb-b624-7f76632fbd27 kind: Event lastTimestamp: "2026-09-16T19:07:50Z" message: Scaled up replica set error-404-isvc-443e6-predictor-569c8dcb6d from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:50Z" name: error-404-isvc-443e6-predictor.18d5e2af7d777c00 namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: 9a4ff635-64f7-4aa0-919e-9f3aaf096270 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:08:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-443e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: 2fbe03bf-4d60-47c5-b36c-77cbae687581 kind: Event lastTimestamp: "2026-09-16T19:08:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:35Z" name: error-404-isvc-443e6-predictor.18d5e2b2fb49347b namespace: kserve-ci-e2e-test resourceVersion: "20681" uid: 72813dad-9dec-49fd-8fa4-f8293955de0e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:08:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-443e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20283" uid: 2fbe03bf-4d60-47c5-b36c-77cbae687581 kind: Event lastTimestamp: "2026-09-16T19:08:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:35Z" name: error-404-isvc-443e6-predictor.18d5e2b2fbb65b24 namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: 6cb0697f-5ad2-43b7-a18b-9726b0f0c5d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-443e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20457" uid: 2fbe03bf-4d60-47c5-b36c-77cbae687581 kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:50Z" name: error-404-isvc-443e6-predictor.18d5e2bd78be8b46 namespace: kserve-ci-e2e-test resourceVersion: "20943" uid: 03a30b6b-7dec-4390-bbc3-3a6e674629f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-443e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20457" uid: 2fbe03bf-4d60-47c5-b36c-77cbae687581 kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:50Z" name: error-404-isvc-443e6-predictor.18d5e2bd79360a9f namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 9be25549-e4a7-4d4b-8c60-23f3c9ab5615 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:07:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-443e6 namespace: kserve-ci-e2e-test resourceVersion: "20262" uid: ab03330b-fc53-4ac9-b6f1-5f050fea3362 kind: Event lastTimestamp: "2026-09-16T19:08:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:08:37Z" name: error-404-isvc-443e6.18d5e2af7c86ae57 namespace: kserve-ci-e2e-test resourceVersion: "20701" uid: f9fb499c-0920-4836-a678-ffa52b950b14 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-443e6 namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: ab03330b-fc53-4ac9-b6f1-5f050fea3362 kind: Event lastTimestamp: "2026-09-16T19:08:37Z" message: InferenceService [error-404-isvc-443e6] is Ready metadata: creationTimestamp: "2026-09-16T19:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:08:37Z" name: error-404-isvc-443e6.18d5e2ba7bb947a7 namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: 070805aa-8a64-4782-ab37-0e5d821cb29c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:10:00.405856Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8 namespace: kserve-ci-e2e-test resourceVersion: "21858" uid: a24533bc-9ebc-4233-8a25-a5d1f42c77de kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-53a10-predictor-765f8bbccb-v7px8 to ip-10-0-142-175.ec2.internal metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:10:00Z" name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8.18d5e2cdc8464dc2 namespace: kserve-ci-e2e-test resourceVersion: "21863" uid: f3903410-89be-44e6-a5f9-20880a6ea0c2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8 namespace: kserve-ci-e2e-test resourceVersion: "21870" uid: a24533bc-9ebc-4233-8a25-a5d1f42c77de kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:10:00Z" name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8.18d5e2cdea6650bf namespace: kserve-ci-e2e-test resourceVersion: "21914" uid: 74aaa46c-d245-4ee1-a30a-7ad63b7614a6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: a24533bc-9ebc-4233-8a25-a5d1f42c77de kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8.18d5e2cdebdcf066 namespace: kserve-ci-e2e-test resourceVersion: "21916" uid: ea60a904-b82c-44be-9623-18a8f2c2b2dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: a24533bc-9ebc-4233-8a25-a5d1f42c77de kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8.18d5e2cdf04d033f namespace: kserve-ci-e2e-test resourceVersion: "21920" uid: 7e29fed6-eb45-422d-a7c5-6d72c634863d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: a24533bc-9ebc-4233-8a25-a5d1f42c77de kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8.18d5e2cdf0d3767d namespace: kserve-ci-e2e-test resourceVersion: "21921" uid: 91fb540b-1b3e-481e-a08d-860609aef20a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: a24533bc-9ebc-4233-8a25-a5d1f42c77de kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8.18d5e2cdf0e0a9e8 namespace: kserve-ci-e2e-test resourceVersion: "21922" uid: 14f4ced2-6186-40ca-bb28-e0ce468b6165 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: a24533bc-9ebc-4233-8a25-a5d1f42c77de kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8.18d5e2cdf42493fe namespace: kserve-ci-e2e-test resourceVersion: "21923" uid: db83320a-d88b-4820-8cb6-b588e206e8a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: a24533bc-9ebc-4233-8a25-a5d1f42c77de kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8.18d5e2cdf4a405ae namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 9aa99aaa-688f-4787-b323-d642f37cdf0d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T19:10:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: a24533bc-9ebc-4233-8a25-a5d1f42c77de kind: Event lastTimestamp: "2026-09-16T19:19:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:10:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:18Z" name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8.18d5e2ce2b371348 namespace: kserve-ci-e2e-test resourceVersion: "26540" uid: 2ab38543-46cb-48a2-9fbb-1dade8df9d94 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: a24533bc-9ebc-4233-8a25-a5d1f42c77de kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:15Z" name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8.18d5e34ef2021d12 namespace: kserve-ci-e2e-test resourceVersion: "26390" uid: 5af53999-6448-4a4e-8937-63897e5d45c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: a24533bc-9ebc-4233-8a25-a5d1f42c77de kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:15Z" name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8.18d5e34ef202fc69 namespace: kserve-ci-e2e-test resourceVersion: "26392" uid: 3b591ea3-8178-4b54-9b81-4e6b41a3975c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: a24533bc-9ebc-4233-8a25-a5d1f42c77de kind: Event lastTimestamp: "2026-09-16T19:19:18Z" message: 'Readiness probe failed: Get "https://10.133.0.43:8643/healthz": dial tcp 10.133.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:19:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:18Z" name: error-404-isvc-53a10-predictor-765f8bbccb-v7px8.18d5e34f9fab4e46 namespace: kserve-ci-e2e-test resourceVersion: "26539" uid: fe8624b0-53ec-4118-b69c-c295e96d3bbc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-53a10-predictor-765f8bbccb namespace: kserve-ci-e2e-test resourceVersion: "21851" uid: 2e9bf8db-dba3-4c7a-8760-5dff5021126f kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: 'Created pod: error-404-isvc-53a10-predictor-765f8bbccb-v7px8' metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:00Z" name: error-404-isvc-53a10-predictor-765f8bbccb.18d5e2cdc79454a1 namespace: kserve-ci-e2e-test resourceVersion: "21859" uid: 8b1ccda1-47a1-4c1d-ad77-92b8f98ee970 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-53a10-predictor namespace: kserve-ci-e2e-test resourceVersion: "21849" uid: a2d04909-a761-45a4-8426-bf82b0d7ed27 kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: Scaled up replica set error-404-isvc-53a10-predictor-765f8bbccb from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:00Z" name: error-404-isvc-53a10-predictor.18d5e2cdc5a8d5de namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: 788845a1-97bd-4758-89cf-a89742f39779 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:10:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-53a10-predictor namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: aa76569f-10d2-4b2e-9e49-18ada2bb7b45 kind: Event lastTimestamp: "2026-09-16T19:10:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:30Z" name: error-404-isvc-53a10-predictor.18d5e2d144506323 namespace: kserve-ci-e2e-test resourceVersion: "22190" uid: e52c305f-2839-4e68-9267-1f3ccf16b964 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:10:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-53a10-predictor namespace: kserve-ci-e2e-test resourceVersion: "21850" uid: aa76569f-10d2-4b2e-9e49-18ada2bb7b45 kind: Event lastTimestamp: "2026-09-16T19:10:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:30Z" name: error-404-isvc-53a10-predictor.18d5e2d144c6e74f namespace: kserve-ci-e2e-test resourceVersion: "22191" uid: d5ca7920-ab47-4af2-a878-8ea909d58c1e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:10:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-53a10-predictor namespace: kserve-ci-e2e-test resourceVersion: "22074" uid: aa76569f-10d2-4b2e-9e49-18ada2bb7b45 kind: Event lastTimestamp: "2026-09-16T19:11:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:10:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:11:15Z" name: error-404-isvc-53a10-predictor.18d5e2d8429f263d namespace: kserve-ci-e2e-test resourceVersion: "22584" uid: c7dadf97-07a4-4181-bfd4-a660f3995aa8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:10:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-53a10-predictor namespace: kserve-ci-e2e-test resourceVersion: "22074" uid: aa76569f-10d2-4b2e-9e49-18ada2bb7b45 kind: Event lastTimestamp: "2026-09-16T19:11:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:10:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:11:15Z" name: error-404-isvc-53a10-predictor.18d5e2d84317403d namespace: kserve-ci-e2e-test resourceVersion: "22585" uid: e21e1520-dc27-477a-b537-34f7e367e8e0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-53a10 namespace: kserve-ci-e2e-test resourceVersion: "21843" uid: 368b57f4-1aa5-4d7e-bc17-9059d1e3b4a9 kind: Event lastTimestamp: "2026-09-16T19:10:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:10:48Z" name: error-404-isvc-53a10.18d5e2cdc5b04ef8 namespace: kserve-ci-e2e-test resourceVersion: "22344" uid: b21014d7-c02d-47ee-9507-9fe76b8a91cb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-53a10 namespace: kserve-ci-e2e-test resourceVersion: "22339" uid: 368b57f4-1aa5-4d7e-bc17-9059d1e3b4a9 kind: Event lastTimestamp: "2026-09-16T19:10:48Z" message: InferenceService [error-404-isvc-53a10] is Ready metadata: creationTimestamp: "2026-09-16T19:10:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:10:48Z" name: error-404-isvc-53a10.18d5e2d8e81f4a3d namespace: kserve-ci-e2e-test resourceVersion: "22340" uid: a1aeac48-cf56-4f24-92e6-955c9e509907 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:19:15.356029Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl namespace: kserve-ci-e2e-test resourceVersion: "26433" uid: 96d7a6a0-de0d-411f-8460-08e50a117de7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7f5a4-predictor-f9d645766-gsvbl to ip-10-0-142-175.ec2.internal metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:19:15Z" name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl.18d5e34efde1e9bf namespace: kserve-ci-e2e-test resourceVersion: "26439" uid: f2dd842f-a774-47ab-9468-821457509cda reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 96d7a6a0-de0d-411f-8460-08e50a117de7 kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7f5a4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:15Z" name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl.18d5e34f08711d31 namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 309b428e-f1c1-4cef-806a-7673799555ca reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: 96d7a6a0-de0d-411f-8460-08e50a117de7 kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:19:16Z" name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl.18d5e34f43d82281 namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: c6e82d89-96ef-4615-9c89-f7664eb68196 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 96d7a6a0-de0d-411f-8460-08e50a117de7 kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl.18d5e34f45610be7 namespace: kserve-ci-e2e-test resourceVersion: "26505" uid: 139c74e4-9ce1-4d84-8039-b53b6819d73d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 96d7a6a0-de0d-411f-8460-08e50a117de7 kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl.18d5e34f48e2b728 namespace: kserve-ci-e2e-test resourceVersion: "26509" uid: e8dedeb3-ac24-41be-a0ee-ea529177be30 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 96d7a6a0-de0d-411f-8460-08e50a117de7 kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl.18d5e34f4960c3bd namespace: kserve-ci-e2e-test resourceVersion: "26511" uid: 3fbf7351-fbf5-4df6-8d9b-5f27fcf805b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 96d7a6a0-de0d-411f-8460-08e50a117de7 kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl.18d5e34f496c12c9 namespace: kserve-ci-e2e-test resourceVersion: "26512" uid: 6ff6f8b1-ffec-4aad-b7a5-a4fe6ee3d542 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 96d7a6a0-de0d-411f-8460-08e50a117de7 kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl.18d5e34f4d38967d namespace: kserve-ci-e2e-test resourceVersion: "26516" uid: 4d06bd96-5f2c-437e-9181-3f50b8ac2039 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 96d7a6a0-de0d-411f-8460-08e50a117de7 kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl.18d5e34f4db68275 namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: 53b656bc-0c1b-4e97-b11f-c3431246ab61 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:19:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 96d7a6a0-de0d-411f-8460-08e50a117de7 kind: Event lastTimestamp: "2026-09-16T19:19:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:53Z" name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl.18d5e34f8f8a5a8b namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: eb1ede2b-b8e4-4fa2-901b-056afcd4bbc7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:36:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 96d7a6a0-de0d-411f-8460-08e50a117de7 kind: Event lastTimestamp: "2026-09-16T19:36:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:36:34Z" name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl.18d5e440eb522d4d namespace: kserve-ci-e2e-test resourceVersion: "34079" uid: 163d1711-6f86-4566-bfe2-02886d68b979 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:36:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 96d7a6a0-de0d-411f-8460-08e50a117de7 kind: Event lastTimestamp: "2026-09-16T19:36:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:36:34Z" name: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl.18d5e440eb529a88 namespace: kserve-ci-e2e-test resourceVersion: "34080" uid: 29e3fc9b-7273-4e4b-889d-e220616dc98d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7f5a4-predictor-f9d645766 namespace: kserve-ci-e2e-test resourceVersion: "26426" uid: 28c2cbb2-9911-49a9-8862-c1d7eb9525c2 kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: 'Created pod: error-404-isvc-7f5a4-predictor-f9d645766-gsvbl' metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:15Z" name: error-404-isvc-7f5a4-predictor-f9d645766.18d5e34efd455f29 namespace: kserve-ci-e2e-test resourceVersion: "26435" uid: e63815d5-b9d8-4084-9599-02cd17b8ab82 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7f5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "26424" uid: 247183ac-cbb6-493a-831a-8b9306e866b7 kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: Scaled up replica set error-404-isvc-7f5a4-predictor-f9d645766 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:15Z" name: error-404-isvc-7f5a4-predictor.18d5e34efb4c7da7 namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: 9c40ce36-2d35-43da-add3-6e6fda55cf8e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7f5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: 19733ab8-9e90-42b7-8922-2d6856121d00 kind: Event lastTimestamp: "2026-09-16T19:19:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:30Z" name: error-404-isvc-7f5a4-predictor.18d5e3527a72f32b namespace: kserve-ci-e2e-test resourceVersion: "26794" uid: 85990a1b-22ac-42c0-9ddc-5be5dc7c69d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7f5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "26425" uid: 19733ab8-9e90-42b7-8922-2d6856121d00 kind: Event lastTimestamp: "2026-09-16T19:19:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:30Z" name: error-404-isvc-7f5a4-predictor.18d5e3527b062450 namespace: kserve-ci-e2e-test resourceVersion: "26795" uid: c08774a9-7ebe-48ee-932c-119c698d4d8e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:19:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7f5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "26793" uid: 19733ab8-9e90-42b7-8922-2d6856121d00 kind: Event lastTimestamp: "2026-09-16T19:20:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:30Z" name: error-404-isvc-7f5a4-predictor.18d5e355fa0e5f01 namespace: kserve-ci-e2e-test resourceVersion: "27385" uid: 7352f663-bbe4-4f71-b984-a5023706a4ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:19:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7f5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "26793" uid: 19733ab8-9e90-42b7-8922-2d6856121d00 kind: Event lastTimestamp: "2026-09-16T19:20:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:30Z" name: error-404-isvc-7f5a4-predictor.18d5e355faa27614 namespace: kserve-ci-e2e-test resourceVersion: "27386" uid: 5169b52f-22e3-48b3-98d1-fd8f004fe940 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7f5a4 namespace: kserve-ci-e2e-test resourceVersion: "26412" uid: 04d4bb58-7cf8-4139-be77-9e612c119dcb kind: Event lastTimestamp: "2026-09-16T19:20:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:20:03Z" name: error-404-isvc-7f5a4.18d5e34efb5a2c39 namespace: kserve-ci-e2e-test resourceVersion: "27091" uid: b652cf4f-a43c-4c18-b9a2-f1c26c61551b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7f5a4 namespace: kserve-ci-e2e-test resourceVersion: "27089" uid: 04d4bb58-7cf8-4139-be77-9e612c119dcb kind: Event lastTimestamp: "2026-09-16T19:20:03Z" message: InferenceService [error-404-isvc-7f5a4] is Ready metadata: creationTimestamp: "2026-09-16T19:20:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:20:03Z" name: error-404-isvc-7f5a4.18d5e35a4b229d17 namespace: kserve-ci-e2e-test resourceVersion: "27090" uid: 80155b27-d2ec-41b5-aaa3-6b310e5f169a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:08:50.065305Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9450b-predictor-59464df8d8-nx628 namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: 188005de-15e8-4854-936b-6e27acfaf44b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9450b-predictor-59464df8d8-nx628 to ip-10-0-142-175.ec2.internal metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:08:50Z" name: error-404-isvc-9450b-predictor-59464df8d8-nx628.18d5e2bd67a6bf8e namespace: kserve-ci-e2e-test resourceVersion: "20893" uid: 1bb1a0fc-2c39-4a9d-b4fe-e28da55cb7b8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9450b-predictor-59464df8d8-nx628 namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: 188005de-15e8-4854-936b-6e27acfaf44b kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:08:50Z" name: error-404-isvc-9450b-predictor-59464df8d8-nx628.18d5e2bd89f7ebbe namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: c6568966-5c5e-4c53-8a86-36621613640a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9450b-predictor-59464df8d8-nx628 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 188005de-15e8-4854-936b-6e27acfaf44b kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:50Z" name: error-404-isvc-9450b-predictor-59464df8d8-nx628.18d5e2bd8b7eeb3c namespace: kserve-ci-e2e-test resourceVersion: "20951" uid: 339ee2e6-4984-4457-89b0-c86263fb0ec3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9450b-predictor-59464df8d8-nx628 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 188005de-15e8-4854-936b-6e27acfaf44b kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:50Z" name: error-404-isvc-9450b-predictor-59464df8d8-nx628.18d5e2bd8e727b58 namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: d1b143d1-b633-45d9-a3d3-f0ffd4f2e440 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9450b-predictor-59464df8d8-nx628 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 188005de-15e8-4854-936b-6e27acfaf44b kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:50Z" name: error-404-isvc-9450b-predictor-59464df8d8-nx628.18d5e2bd8ef11777 namespace: kserve-ci-e2e-test resourceVersion: "20954" uid: c0247c1a-c3be-4bba-99cd-66e1a83da55f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9450b-predictor-59464df8d8-nx628 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 188005de-15e8-4854-936b-6e27acfaf44b kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:50Z" name: error-404-isvc-9450b-predictor-59464df8d8-nx628.18d5e2bd8efc9c97 namespace: kserve-ci-e2e-test resourceVersion: "20956" uid: e690c2c4-3985-4830-8e8b-8747f6588dfd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9450b-predictor-59464df8d8-nx628 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 188005de-15e8-4854-936b-6e27acfaf44b kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:50Z" name: error-404-isvc-9450b-predictor-59464df8d8-nx628.18d5e2bd92083668 namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: faa303ac-0917-40bf-bca7-7440425745a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9450b-predictor-59464df8d8-nx628 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 188005de-15e8-4854-936b-6e27acfaf44b kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:50Z" name: error-404-isvc-9450b-predictor-59464df8d8-nx628.18d5e2bd928fd354 namespace: kserve-ci-e2e-test resourceVersion: "20960" uid: 3e51da3d-7bca-4e33-aea2-272b0bc22143 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:08:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9450b-predictor-59464df8d8-nx628 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 188005de-15e8-4854-936b-6e27acfaf44b kind: Event lastTimestamp: "2026-09-16T19:09:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:27Z" name: error-404-isvc-9450b-predictor-59464df8d8-nx628.18d5e2bdc938f098 namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: 5b2db80b-057d-4a70-9e34-f7b149995a12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9450b-predictor-59464df8d8-nx628 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 188005de-15e8-4854-936b-6e27acfaf44b kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:00Z" name: error-404-isvc-9450b-predictor-59464df8d8-nx628.18d5e2cdb8f5d03c namespace: kserve-ci-e2e-test resourceVersion: "21809" uid: 923dfefb-ae4f-44b7-8927-ec12bcafe6cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9450b-predictor-59464df8d8-nx628 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 188005de-15e8-4854-936b-6e27acfaf44b kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:00Z" name: error-404-isvc-9450b-predictor-59464df8d8-nx628.18d5e2cdb8f65a7d namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: 4d736c33-b469-4e8f-abb9-07c46ba677ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9450b-predictor-59464df8d8-nx628 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 188005de-15e8-4854-936b-6e27acfaf44b kind: Event lastTimestamp: "2026-09-16T19:10:02Z" message: 'Readiness probe failed: Get "https://10.133.0.41:8643/healthz": dial tcp 10.133.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:10:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:02Z" name: error-404-isvc-9450b-predictor-59464df8d8-nx628.18d5e2ce51677c3c namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: 195b3b2e-4a6a-4fd9-b4d9-8d327962d140 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9450b-predictor-59464df8d8 namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: 6887588a-0426-43eb-b0ce-46e85767d655 kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: 'Created pod: error-404-isvc-9450b-predictor-59464df8d8-nx628' metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:50Z" name: error-404-isvc-9450b-predictor-59464df8d8.18d5e2bd671556b5 namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: ad0ba3db-d46b-4792-b3b5-5005456eef89 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9450b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: d7dc72e6-707f-4707-8fc8-a2d6d93d8b49 kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: Scaled up replica set error-404-isvc-9450b-predictor-59464df8d8 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:50Z" name: error-404-isvc-9450b-predictor.18d5e2bd6516aa00 namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: d956179f-d3ea-4f38-9673-3e519fee1bcd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9450b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: d9b08d8c-9164-4bab-a37f-768460dd0d2f kind: Event lastTimestamp: "2026-09-16T19:09:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:09:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:05Z" name: error-404-isvc-9450b-predictor.18d5e2c0e3e940a2 namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 10b86414-08e7-478a-9e2b-aa2074d27527 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9450b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: d9b08d8c-9164-4bab-a37f-768460dd0d2f kind: Event lastTimestamp: "2026-09-16T19:09:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:09:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:05Z" name: error-404-isvc-9450b-predictor.18d5e2c0e46669ef namespace: kserve-ci-e2e-test resourceVersion: "21248" uid: 35cb7e98-88f8-4757-8c98-aa99deb2a719 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:09:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9450b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: d9b08d8c-9164-4bab-a37f-768460dd0d2f kind: Event lastTimestamp: "2026-09-16T19:09:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:50Z" name: error-404-isvc-9450b-predictor.18d5e2c463393da8 namespace: kserve-ci-e2e-test resourceVersion: "21678" uid: d76b29b8-ef28-40d8-92f2-17f0510a4b0b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:09:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9450b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21247" uid: d9b08d8c-9164-4bab-a37f-768460dd0d2f kind: Event lastTimestamp: "2026-09-16T19:09:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:50Z" name: error-404-isvc-9450b-predictor.18d5e2c463ab9e28 namespace: kserve-ci-e2e-test resourceVersion: "21679" uid: 9c9713d9-f970-46ea-b5e2-c8d790d73308 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9450b namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 5a429f44-eb17-4011-b9c8-0fa3bdf8c2ac kind: Event lastTimestamp: "2026-09-16T19:09:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:09:37Z" name: error-404-isvc-9450b.18d5e2bd6520161e namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: 7a4acc35-e2bd-4d48-bf42-60b38e983df4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9450b namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: 5a429f44-eb17-4011-b9c8-0fa3bdf8c2ac kind: Event lastTimestamp: "2026-09-16T19:09:37Z" message: InferenceService [error-404-isvc-9450b] is Ready metadata: creationTimestamp: "2026-09-16T19:09:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:09:37Z" name: error-404-isvc-9450b.18d5e2c885397fb1 namespace: kserve-ci-e2e-test resourceVersion: "21525" uid: af7db0d0-96b7-44a6-bd07-71e9bd0a05d2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:58:35.198643Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq namespace: kserve-ci-e2e-test resourceVersion: "15833" uid: 494bb4ca-014f-4713-aff2-6456f8b5a03e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq to ip-10-0-142-175.ec2.internal metadata: creationTimestamp: "2026-09-16T18:58:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:58:35Z" name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq.18d5e22e3ebdfd27 namespace: kserve-ci-e2e-test resourceVersion: "15838" uid: 6893fe6d-a9bf-4ef6-bcaf-53a1744d15de reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 494bb4ca-014f-4713-aff2-6456f8b5a03e kind: Event lastTimestamp: "2026-09-16T18:58:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-9a5c2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T18:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:35Z" name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq.18d5e22e480a336c namespace: kserve-ci-e2e-test resourceVersion: "15852" uid: e85896d8-b31c-4690-9e71-3106aecf17dc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: 494bb4ca-014f-4713-aff2-6456f8b5a03e kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:58:36Z" name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq.18d5e22e84af2d73 namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: e1fdebd5-2528-46db-92f9-396a2ee1da72 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 494bb4ca-014f-4713-aff2-6456f8b5a03e kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:36Z" name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq.18d5e22e8649277a namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: e6a94946-ae37-4640-b2d2-5a032bddb0f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 494bb4ca-014f-4713-aff2-6456f8b5a03e kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:36Z" name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq.18d5e22e89b96efa namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: d2119257-d2ab-4df7-a74e-7c6d50d5082b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 494bb4ca-014f-4713-aff2-6456f8b5a03e kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:36Z" name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq.18d5e22e8a3223c0 namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: 7119d08a-e48e-4b75-957d-d6f0f547964f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 494bb4ca-014f-4713-aff2-6456f8b5a03e kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:36Z" name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq.18d5e22e8a3ed0d2 namespace: kserve-ci-e2e-test resourceVersion: "15881" uid: 26e5f9c1-6bc8-48f2-957c-03c3c43f1541 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 494bb4ca-014f-4713-aff2-6456f8b5a03e kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:36Z" name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq.18d5e22e8d58ad9d namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: 4e088c24-dde0-4a1e-8b3b-5bb277591b0b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 494bb4ca-014f-4713-aff2-6456f8b5a03e kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:36Z" name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq.18d5e22e8dce3d37 namespace: kserve-ci-e2e-test resourceVersion: "15883" uid: 7abe4085-5e12-457f-88bf-7ee71a7b8a95 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:58:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 494bb4ca-014f-4713-aff2-6456f8b5a03e kind: Event lastTimestamp: "2026-09-16T18:59:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:13Z" name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq.18d5e22ed4172578 namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: d27fe040-d729-4dd1-abfb-be017d2014a0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 494bb4ca-014f-4713-aff2-6456f8b5a03e kind: Event lastTimestamp: "2026-09-16T19:07:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:50Z" name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq.18d5e2af78c9c465 namespace: kserve-ci-e2e-test resourceVersion: "20267" uid: e7cda306-fa7b-4a08-944b-ccce7151e35d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq namespace: kserve-ci-e2e-test resourceVersion: "15835" uid: 494bb4ca-014f-4713-aff2-6456f8b5a03e kind: Event lastTimestamp: "2026-09-16T19:07:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:50Z" name: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq.18d5e2af78ca9ad6 namespace: kserve-ci-e2e-test resourceVersion: "20270" uid: a1c93092-c6bb-4e07-9c40-564cacaa2ec7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9a5c2-predictor-6999f87ffb namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: a5e352c5-ca5d-4dac-b25c-e81775d1e568 kind: Event lastTimestamp: "2026-09-16T18:58:35Z" message: 'Created pod: error-404-isvc-9a5c2-predictor-6999f87ffb-m76lq' metadata: creationTimestamp: "2026-09-16T18:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:35Z" name: error-404-isvc-9a5c2-predictor-6999f87ffb.18d5e22e3e460340 namespace: kserve-ci-e2e-test resourceVersion: "15834" uid: 59f38fa7-0722-464e-b45d-543f29487fd9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9a5c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15824" uid: 60f95f15-655a-446c-a1d7-30324feb9106 kind: Event lastTimestamp: "2026-09-16T18:58:35Z" message: Scaled up replica set error-404-isvc-9a5c2-predictor-6999f87ffb from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:35Z" name: error-404-isvc-9a5c2-predictor.18d5e22e3c961828 namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: e53fe290-5f37-406f-8c74-6b277504693c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9a5c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: 5b5e5505-14ff-4400-ac9f-49132159ec13 kind: Event lastTimestamp: "2026-09-16T18:58:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:50Z" name: error-404-isvc-9a5c2-predictor.18d5e231bb1f07c7 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: 3d448e59-adbd-49b8-93e5-85437b52b51c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9a5c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15825" uid: 5b5e5505-14ff-4400-ac9f-49132159ec13 kind: Event lastTimestamp: "2026-09-16T18:58:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:50Z" name: error-404-isvc-9a5c2-predictor.18d5e231bb827bb0 namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: e53e2946-23a2-4721-8a97-ef3249cd2e99 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:59:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9a5c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: 5b5e5505-14ff-4400-ac9f-49132159ec13 kind: Event lastTimestamp: "2026-09-16T19:00:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:05Z" name: error-404-isvc-9a5c2-predictor.18d5e2353a3500b2 namespace: kserve-ci-e2e-test resourceVersion: "16967" uid: 2cd2fb69-b221-4d0e-8e1c-9bbdfefb3a03 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:59:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9a5c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: 5b5e5505-14ff-4400-ac9f-49132159ec13 kind: Event lastTimestamp: "2026-09-16T19:00:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:05Z" name: error-404-isvc-9a5c2-predictor.18d5e2353ab58026 namespace: kserve-ci-e2e-test resourceVersion: "16968" uid: 2478cff4-3498-4053-a72f-f0cb5dbbec01 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:58:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9a5c2 namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: 9d58a0d1-735f-49ba-8608-650d717b7c7e kind: Event lastTimestamp: "2026-09-16T18:59:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:59:23Z" name: error-404-isvc-9a5c2.18d5e22e3ca8d735 namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: f5999ba9-8176-4940-84e2-3ad86fd23bb7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9a5c2 namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: 9d58a0d1-735f-49ba-8608-650d717b7c7e kind: Event lastTimestamp: "2026-09-16T18:59:23Z" message: InferenceService [error-404-isvc-9a5c2] is Ready metadata: creationTimestamp: "2026-09-16T18:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:59:23Z" name: error-404-isvc-9a5c2.18d5e2398ff2e192 namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 872930a2-3483-49c0-aa39-43b8ceee8def reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:59:35.237307Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj namespace: kserve-ci-e2e-test resourceVersion: "16641" uid: 0d258cd9-b127-46e1-ae09-50ddd25c0eff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a895e-predictor-8c7b557fb-l68lj to ip-10-0-142-175.ec2.internal metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:59:35Z" name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj.18d5e23c39534cc7 namespace: kserve-ci-e2e-test resourceVersion: "16647" uid: 1ddb3282-948b-415e-9be9-ad6dcc9d92e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj namespace: kserve-ci-e2e-test resourceVersion: "16657" uid: 0d258cd9-b127-46e1-ae09-50ddd25c0eff kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:59:35Z" name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj.18d5e23c5b75f888 namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: 5eb2f143-879e-4b8d-9d4e-20f2fd95be8c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 0d258cd9-b127-46e1-ae09-50ddd25c0eff kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj.18d5e23c5ce28441 namespace: kserve-ci-e2e-test resourceVersion: "16683" uid: fdf2efb8-bf6b-43f7-b481-861869249ade reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 0d258cd9-b127-46e1-ae09-50ddd25c0eff kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj.18d5e23c60158e5d namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: a7c38943-bc33-44d1-85c2-087b28ec49c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 0d258cd9-b127-46e1-ae09-50ddd25c0eff kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj.18d5e23c608c9513 namespace: kserve-ci-e2e-test resourceVersion: "16686" uid: 2d07dc4d-0f09-4438-b072-76837ee6fea1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 0d258cd9-b127-46e1-ae09-50ddd25c0eff kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj.18d5e23c609a696a namespace: kserve-ci-e2e-test resourceVersion: "16687" uid: a3b39304-b0fc-41bc-a6ca-80a74f492565 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 0d258cd9-b127-46e1-ae09-50ddd25c0eff kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj.18d5e23c637eb560 namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 1f7d66c9-91be-4d49-aabf-98f2530dcfb8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 0d258cd9-b127-46e1-ae09-50ddd25c0eff kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj.18d5e23c63f30df8 namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: 2124d369-0c0d-42cc-9c11-ce3634fdedd9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T18:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 0d258cd9-b127-46e1-ae09-50ddd25c0eff kind: Event lastTimestamp: "2026-09-16T19:08:53Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:53Z" name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj.18d5e23ca35957db namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: 065bf371-c5cc-410b-a5fe-7fac11d645c1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 0d258cd9-b127-46e1-ae09-50ddd25c0eff kind: Event lastTimestamp: "2026-09-16T19:08:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:49Z" name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj.18d5e2bd5c1f9629 namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: e14728c5-ab6a-4503-b030-85687bec6d0f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 0d258cd9-b127-46e1-ae09-50ddd25c0eff kind: Event lastTimestamp: "2026-09-16T19:08:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:49Z" name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj.18d5e2bd5c204f8d namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: 091fe080-0455-48a7-b9bd-4daf1358eb33 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: 0d258cd9-b127-46e1-ae09-50ddd25c0eff kind: Event lastTimestamp: "2026-09-16T19:08:53Z" message: 'Readiness probe failed: Get "https://10.133.0.39:8643/healthz": dial tcp 10.133.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:08:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:53Z" name: error-404-isvc-a895e-predictor-8c7b557fb-l68lj.18d5e2be17d8bd03 namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: 90d9ae68-cfd5-4a4f-8dde-cdbe4df7441c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a895e-predictor-8c7b557fb namespace: kserve-ci-e2e-test resourceVersion: "16632" uid: af66e7cc-3a40-4f46-bc79-5c0959d8b25a kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: 'Created pod: error-404-isvc-a895e-predictor-8c7b557fb-l68lj' metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:35Z" name: error-404-isvc-a895e-predictor-8c7b557fb.18d5e23c38cde378 namespace: kserve-ci-e2e-test resourceVersion: "16643" uid: 181f2178-a2b0-4664-8ddd-deb645d5e7f6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a895e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16628" uid: ce4ec817-f982-456d-a8fb-7fe28b2c7d1e kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Scaled up replica set error-404-isvc-a895e-predictor-8c7b557fb from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:35Z" name: error-404-isvc-a895e-predictor.18d5e23c3716e386 namespace: kserve-ci-e2e-test resourceVersion: "16636" uid: 9223422e-974b-4751-9de9-6d79f07f3895 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:59:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a895e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16631" uid: 34260732-092c-449d-8170-19098bb6e77a kind: Event lastTimestamp: "2026-09-16T19:00:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:05Z" name: error-404-isvc-a895e-predictor.18d5e23fb5de1aac namespace: kserve-ci-e2e-test resourceVersion: "16965" uid: 3bf6ed43-cf3d-4f67-8259-4ef9ecbdd276 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:59:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a895e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16631" uid: 34260732-092c-449d-8170-19098bb6e77a kind: Event lastTimestamp: "2026-09-16T19:00:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:05Z" name: error-404-isvc-a895e-predictor.18d5e23fb657a534 namespace: kserve-ci-e2e-test resourceVersion: "16966" uid: 06080756-5d31-4f93-bb58-5be31f21312b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:00:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a895e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 34260732-092c-449d-8170-19098bb6e77a kind: Event lastTimestamp: "2026-09-16T19:00:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:50Z" name: error-404-isvc-a895e-predictor.18d5e246b44008b2 namespace: kserve-ci-e2e-test resourceVersion: "17352" uid: 0e0fe8dc-7a59-4a25-8a43-2b38d881906c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:00:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a895e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 34260732-092c-449d-8170-19098bb6e77a kind: Event lastTimestamp: "2026-09-16T19:00:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:50Z" name: error-404-isvc-a895e-predictor.18d5e246b4c701ed namespace: kserve-ci-e2e-test resourceVersion: "17353" uid: 3b2a09e2-f1dc-43a2-9b48-a78222d39982 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a895e namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 5008c8b8-7aed-4f2e-93fc-c064b1db51e9 kind: Event lastTimestamp: "2026-09-16T19:00:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:00:23Z" name: error-404-isvc-a895e.18d5e23c3711939b namespace: kserve-ci-e2e-test resourceVersion: "17111" uid: e9e60244-1a6a-4c36-8d19-8acefc0d4cc3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a895e namespace: kserve-ci-e2e-test resourceVersion: "17109" uid: 5008c8b8-7aed-4f2e-93fc-c064b1db51e9 kind: Event lastTimestamp: "2026-09-16T19:00:23Z" message: InferenceService [error-404-isvc-a895e] is Ready metadata: creationTimestamp: "2026-09-16T19:00:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:00:23Z" name: error-404-isvc-a895e.18d5e2475f697a5f namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 3e2a6dff-586f-4657-abed-e06741b2f591 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:57:05.072137Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14776" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-dc809-predictor-59c6f6455b-2r8hp to ip-10-0-142-175.ec2.internal metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:57:05Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e21942c8ac30 namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: 6ed5b88f-f0bd-4ee7-8dd7-9b356801abad reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14793" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:57:05Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e2196526f128 namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: 4cced97b-c3b2-4c08-8d8c-3c2383c3beff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1962" metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:05Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e21966aeb64e namespace: kserve-ci-e2e-test resourceVersion: "14864" uid: 7b019c7e-e46c-43e9-aba1-3b256f8cd2b3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: "2026-09-16T18:57:17Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1962" in 11.557s (11.557s including waiting). Image size: 1357342804 bytes.' metadata: creationTimestamp: "2026-09-16T18:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:17Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e21c1796c248 namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: 006317e7-9561-45ec-86a0-2a531a75118e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: "2026-09-16T18:57:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:17Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e21c1b7a0100 namespace: kserve-ci-e2e-test resourceVersion: "15006" uid: c51e76db-4486-407b-83ed-56b1c179cd60 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: "2026-09-16T18:57:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:17Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e21c1bfa56d5 namespace: kserve-ci-e2e-test resourceVersion: "15007" uid: 33a5b70a-e98c-40df-98be-b9084bb3f63f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: "2026-09-16T18:57:17Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-16T18:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:17Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e21c1c088a4b namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: 6a13e6b7-0310-497b-af8e-28a89da089dc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: "2026-09-16T18:57:19Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.215s (2.215s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-16T18:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:19Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e21ca01a54b4 namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: cf8ea463-2ec6-4ace-b5ea-4ee18d71c886 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: "2026-09-16T18:57:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:19Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e21ca3fbc4b1 namespace: kserve-ci-e2e-test resourceVersion: "15028" uid: b6867e6b-364f-4e70-8cb5-70b143400c64 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: "2026-09-16T18:57:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:19Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e21ca478fb8c namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: 13ed038b-e221-4681-91fe-87b68127bed6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T18:57:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: "2026-09-16T18:58:37Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:57:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:37Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e21d0c68536c namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 4eb9007d-78fc-4a54-8853-db2c8a00ef0d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: "2026-09-16T18:58:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:35Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e22e336870a9 namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: ee79ebbd-5c58-42bc-8ee8-85fce55b0990 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: "2026-09-16T18:58:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:35Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e22e3368ef1e namespace: kserve-ci-e2e-test resourceVersion: "15793" uid: 39bd60ec-8709-4461-bbef-d39c72ef4867 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: bfbceae1-9896-4df9-af4c-05ecff945df6 kind: Event lastTimestamp: "2026-09-16T18:58:37Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:37Z" name: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp.18d5e22ebe8ef2cd namespace: kserve-ci-e2e-test resourceVersion: "15899" uid: cb03439d-fde1-4d43-b1eb-85f925d5a5f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-175.ec2.internal source: component: kubelet host: ip-10-0-142-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-dc809-predictor-59c6f6455b namespace: kserve-ci-e2e-test resourceVersion: "14768" uid: 2d081e84-5218-40cf-9a3b-bc4fffe24aae kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: 'Created pod: error-404-isvc-dc809-predictor-59c6f6455b-2r8hp' metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:05Z" name: error-404-isvc-dc809-predictor-59c6f6455b.18d5e21941b79efb namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: 9d454bff-631a-4786-814b-b669a0116d66 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-dc809-predictor namespace: kserve-ci-e2e-test resourceVersion: "14766" uid: 702851ed-dcb4-457d-9520-df2a85a0fec5 kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: Scaled up replica set error-404-isvc-dc809-predictor-59c6f6455b from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:05Z" name: error-404-isvc-dc809-predictor.18d5e2193f051372 namespace: kserve-ci-e2e-test resourceVersion: "14770" uid: 1b7bd612-cfca-4c47-806a-4a9d60bf5979 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:57:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dc809-predictor namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: f9bdd026-5bb2-4fc8-b47a-b0cd4a95674c kind: Event lastTimestamp: "2026-09-16T18:58:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:05Z" name: error-404-isvc-dc809-predictor.18d5e21cbd97c413 namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: 8ad357c7-aad0-4473-83e9-9b3555ef51aa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:57:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dc809-predictor namespace: kserve-ci-e2e-test resourceVersion: "14767" uid: f9bdd026-5bb2-4fc8-b47a-b0cd4a95674c kind: Event lastTimestamp: "2026-09-16T18:58:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:05Z" name: error-404-isvc-dc809-predictor.18d5e21cbe090afc namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 6320e02f-08ef-4936-a3f6-aa47ecebb602 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:57:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dc809-predictor namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: f9bdd026-5bb2-4fc8-b47a-b0cd4a95674c kind: Event lastTimestamp: "2026-09-16T18:58:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:57:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:20Z" name: error-404-isvc-dc809-predictor.18d5e223bb697937 namespace: kserve-ci-e2e-test resourceVersion: "15583" uid: 8f5b172b-0f79-46ad-8bbb-353581aebd1a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:57:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dc809-predictor namespace: kserve-ci-e2e-test resourceVersion: "15039" uid: f9bdd026-5bb2-4fc8-b47a-b0cd4a95674c kind: Event lastTimestamp: "2026-09-16T18:58:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:57:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:20Z" name: error-404-isvc-dc809-predictor.18d5e223bbbe15d8 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: df2d97e4-0dbd-4592-a21b-b20564ae7d05 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dc809 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 4107d524-a60b-4acf-b595-828366753e62 kind: Event lastTimestamp: "2026-09-16T18:58:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:07Z" name: error-404-isvc-dc809.18d5e2193f0e0d98 namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: e559bb3c-203c-41b6-8561-5115951a94a6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dc809 namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: 4107d524-a60b-4acf-b595-828366753e62 kind: Event lastTimestamp: "2026-09-16T18:58:07Z" message: InferenceService [error-404-isvc-dc809] is Ready metadata: creationTimestamp: "2026-09-16T18:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:07Z" name: error-404-isvc-dc809.18d5e227ca0beaf6 namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: df8b145a-5108-47c2-b3f4-0d4e6fb2ca5d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:57:04.797115Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14722" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 to ip-10-0-134-125.ec2.internal metadata: creationTimestamp: "2026-09-16T18:57:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:57:04Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e21932642c65 namespace: kserve-ci-e2e-test resourceVersion: "14726" uid: 9e627002-41d4-4d7d-bb7e-7511152e2952 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:05Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e2193f32c58f namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: e63f2e22-85f6-468f-8526-9e8348d19ec5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14733" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:57:05Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e219784381b6 namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: e418faf2-d692-4c3e-8d89-ed40dd3b4ad4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a69470ab71f96748343c847bfecfdfb03e198759f46531295e483d003e69266a" already present on machine metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:05Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e21979d3f16f namespace: kserve-ci-e2e-test resourceVersion: "14877" uid: 8fc0c868-75cc-4139-af7c-dc5f878e7efc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:57:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:57:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:06Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e2197e07b0b7 namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: d16fef04-dd04-4e0b-8059-d73f77c19953 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:57:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:57:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:06Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e2197e888b00 namespace: kserve-ci-e2e-test resourceVersion: "14881" uid: 889023e7-aad2-4af0-815c-3b0e622e1e4e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:57:10Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1962" metadata: creationTimestamp: "2026-09-16T18:57:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:10Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e21a71443a63 namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: f02478cf-3742-406b-ad81-34caafe250bf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:57:38Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1962" in 27.858s (27.858s including waiting). Image size: 1580849567 bytes.' metadata: creationTimestamp: "2026-09-16T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:38Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e220edc28af7 namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: fbf60f1e-ed84-4394-b1a4-89dbb6e072db reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:57:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:38Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e220fdeb7e44 namespace: kserve-ci-e2e-test resourceVersion: "15202" uid: 25e56070-3c9f-4e98-8d05-2b4df7dbff94 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:57:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:38Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e220ff3d974d namespace: kserve-ci-e2e-test resourceVersion: "15204" uid: b693a666-356f-467a-bece-65f4c069896b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:57:38Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-16T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:38Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e220ff4bb2b8 namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: 36427e13-1b1f-4df0-b75b-db02bffe0fe5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:57:41Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.938s (2.938s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-16T18:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:41Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e221ae6de959 namespace: kserve-ci-e2e-test resourceVersion: "15233" uid: 6a6bdb9c-c3f6-4977-9c97-e1e3f7402ea8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:57:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:41Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e221b2f28041 namespace: kserve-ci-e2e-test resourceVersion: "15238" uid: bd35fe70-806a-424d-b90c-5f3c9b803c74 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:57:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:41Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e221b393a9e9 namespace: kserve-ci-e2e-test resourceVersion: "15241" uid: 929c9a22-9875-4e06-a880-1a9e12266680 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T18:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:59:37Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:37Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e221b67126c6 namespace: kserve-ci-e2e-test resourceVersion: "16711" uid: 5eed3295-c293-46ca-a0d9-840eb506a7d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e23c2b96e7dc namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 25454bbc-be18-4b5f-9b08-b3d47d61752b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e23c2b976c23 namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: 7e39906f-20e9-4715-9a07-b0993328a924 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6 namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: b1451641-e30a-461d-a14f-fa82d5243955 kind: Event lastTimestamp: "2026-09-16T18:59:37Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:37Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6.18d5e23cb9210280 namespace: kserve-ci-e2e-test resourceVersion: "16710" uid: e6267be0-0817-43b6-88f7-fa8db01e87e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-597b9ffcf8 namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: cb53e63d-b15d-4b46-9a8f-b935628a81bc kind: Event lastTimestamp: "2026-09-16T18:57:04Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-597b9ffcf8-2zzm6' metadata: creationTimestamp: "2026-09-16T18:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:04Z" name: isvc-sklearn-graph-1-predictor-597b9ffcf8.18d5e21931e92776 namespace: kserve-ci-e2e-test resourceVersion: "14723" uid: 2812c1b3-8760-47cc-9208-1bb6922e3c95 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: 4bee47ca-7c94-47de-8def-ffc13d30f1f5 kind: Event lastTimestamp: "2026-09-16T18:57:04Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-597b9ffcf8 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:04Z" name: isvc-sklearn-graph-1-predictor.18d5e2192d49dd4e namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: cbcfd847-f8cf-402c-bc64-010fff318004 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:57:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14727" uid: 78c8ea86-8610-4673-866f-7a4e79808de3 kind: Event lastTimestamp: "2026-09-16T18:58:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:19Z" name: isvc-sklearn-graph-1-predictor.18d5e21cb38b687b namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 5ce66fe8-7c28-4058-94cb-a8d8bed43879 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:57:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14727" uid: 78c8ea86-8610-4673-866f-7a4e79808de3 kind: Event lastTimestamp: "2026-09-16T18:58:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:19Z" name: isvc-sklearn-graph-1-predictor.18d5e21cb3f9c2d2 namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: f9092a37-4a78-4e63-aa32-f71fd03c3b33 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:58:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: 78c8ea86-8610-4673-866f-7a4e79808de3 kind: Event lastTimestamp: "2026-09-16T18:59:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:58:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:20Z" name: isvc-sklearn-graph-1-predictor.18d5e22e2fc4d6a2 namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 89271f86-359a-4540-90d8-170f0db8b01f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:58:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15032" uid: 78c8ea86-8610-4673-866f-7a4e79808de3 kind: Event lastTimestamp: "2026-09-16T18:59:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:58:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:20Z" name: isvc-sklearn-graph-1-predictor.18d5e22e302b36a7 namespace: kserve-ci-e2e-test resourceVersion: "16331" uid: 689763f6-90ab-4915-81ba-0801b338ef59 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-16T18:57:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 299b1fe2-18ca-481c-8d51-598664d1185e kind: Event lastTimestamp: "2026-09-16T18:58:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:57Z" name: isvc-sklearn-graph-1.18d5e219334cc338 namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 882c7b51-3978-456e-a21b-64b9d9065512 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: 299b1fe2-18ca-481c-8d51-598664d1185e kind: Event lastTimestamp: "2026-09-16T18:58:57Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-09-16T18:58:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:57Z" name: isvc-sklearn-graph-1.18d5e2336ec16adc namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 6e4078db-9878-4ba7-9496-4b6f595f62f4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:57:05.374640Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-99489b495-hg58r to ip-10-0-134-125.ec2.internal metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:57:05Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e21954d08aa7 namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 8d3fb2a0-d6a3-4959-bc46-6fb4491090e5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:05Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e2196357bd3d namespace: kserve-ci-e2e-test resourceVersion: "14859" uid: 4ab7f714-3805-49be-a725-eb01aafb4e5d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14846" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:57:06Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:57:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:57:06Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e2199be729d7 namespace: kserve-ci-e2e-test resourceVersion: "14898" uid: f521d8ca-3d0b-4d0d-9086-27581b4f3bd4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:57:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a69470ab71f96748343c847bfecfdfb03e198759f46531295e483d003e69266a" already present on machine metadata: creationTimestamp: "2026-09-16T18:57:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:06Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e2199dc9ea20 namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 108ed880-ce10-488c-995a-7596f0570c44 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:57:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:57:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:06Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e219a14b7fc0 namespace: kserve-ci-e2e-test resourceVersion: "14902" uid: df1711ec-3426-4c53-9efe-2b5f8b4a22ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:57:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:57:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:06Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e219a1cae49a namespace: kserve-ci-e2e-test resourceVersion: "14903" uid: 0c7c3042-8a37-4736-ace8-750d316ed727 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:57:11Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1962" metadata: creationTimestamp: "2026-09-16T18:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:11Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e21aadc56dcd namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 5a2d81c9-ca1f-4cbe-a98b-2a02ca562b2b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:57:38Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1962" in 27.048s (27.048s including waiting). Image size: 1580849567 bytes.' metadata: creationTimestamp: "2026-09-16T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:38Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e220fa03190b namespace: kserve-ci-e2e-test resourceVersion: "15201" uid: ff7d6e1a-dced-4fc4-87fa-529780679c17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:57:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:38Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e22100ec0714 namespace: kserve-ci-e2e-test resourceVersion: "15206" uid: 09e9a03d-19af-4d6d-8735-2825c7eb4bd4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:57:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:38Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e22102383168 namespace: kserve-ci-e2e-test resourceVersion: "15208" uid: 56ec66e6-f051-427a-8b30-48d78835c026 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:57:38Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-16T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:38Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e221024632bc namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 3b25da70-58b5-42f8-a286-462ef1981b19 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:57:41Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.899s (2.899s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-16T18:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:41Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e221af1abe75 namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: ca1d5794-2c52-4144-884e-0076cfa47991 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:57:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:41Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e221b2ec9912 namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: 6e0cbce1-15bd-4c03-9076-8bb4aece87f8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:57:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:41Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e221b393358c namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: cfc47ed6-6ab4-411b-a58f-d16c5412161b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T18:57:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:59:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:38Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e221f2a0000d namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: c2bc70bf-c093-47c8-b76b-f055aadb01fc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:59:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:34Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e23c28c71ee8 namespace: kserve-ci-e2e-test resourceVersion: "16535" uid: 6e80b355-2cea-4b2e-b230-d3854305d725 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:59:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:34Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e23c28c79ffc namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 60b6f70c-6e55-4056-9a23-6a1ff96e83bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-99489b495-hg58r namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d21f1d33-0f7f-4d9f-9d55-f04c961d3161 kind: Event lastTimestamp: "2026-09-16T18:59:38Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:38Z" name: isvc-sklearn-graph-2-predictor-99489b495-hg58r.18d5e23cf50164a6 namespace: kserve-ci-e2e-test resourceVersion: "16721" uid: c975c720-a0e8-45ef-ac1b-1695ddddb4a4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-99489b495 namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: d68cec32-50c7-4f7e-babe-e60c06ffa20c kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-99489b495-hg58r' metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:05Z" name: isvc-sklearn-graph-2-predictor-99489b495.18d5e2195433456f namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: d51181a4-22f1-48d1-99f1-f7616a6de6ad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14825" uid: 1f5787b5-5255-4430-b28f-5e2cb2d993ce kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-99489b495 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:05Z" name: isvc-sklearn-graph-2-predictor.18d5e2195106bc0c namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: 587dfb1e-d481-4c10-bac7-2c234c1eb203 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:57:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: e7ee5b23-14b9-485f-aecb-a58f9e479908 kind: Event lastTimestamp: "2026-09-16T18:58:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:05Z" name: isvc-sklearn-graph-2-predictor.18d5e21ccf934190 namespace: kserve-ci-e2e-test resourceVersion: "15450" uid: a3495a56-87bc-44f4-bab1-a35edcabc56e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T18:57:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: e7ee5b23-14b9-485f-aecb-a58f9e479908 kind: Event lastTimestamp: "2026-09-16T18:58:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:05Z" name: isvc-sklearn-graph-2-predictor.18d5e21cd00370bd namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: 98c03462-328c-44c9-b1e8-2429d86131f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:58:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: e7ee5b23-14b9-485f-aecb-a58f9e479908 kind: Event lastTimestamp: "2026-09-16T18:59:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:58:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:20Z" name: isvc-sklearn-graph-2-predictor.18d5e22acb00985e namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: a1bc34d1-a12e-4464-82b6-6fcd6d321a58 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:58:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15051" uid: e7ee5b23-14b9-485f-aecb-a58f9e479908 kind: Event lastTimestamp: "2026-09-16T18:59:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:58:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:20Z" name: isvc-sklearn-graph-2-predictor.18d5e22acb77795b namespace: kserve-ci-e2e-test resourceVersion: "16342" uid: 971c198b-f4e4-459b-8784-4efb777bc3c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: f68a98f0-0be9-42b5-b0b2-4fbcdaec02e9 kind: Event lastTimestamp: "2026-09-16T18:58:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:58Z" name: isvc-sklearn-graph-2.18d5e219510f76fe namespace: kserve-ci-e2e-test resourceVersion: "16107" uid: 3e2f0fb1-5213-427b-9cfe-23d84f94ddc4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: f68a98f0-0be9-42b5-b0b2-4fbcdaec02e9 kind: Event lastTimestamp: "2026-09-16T18:58:58Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-09-16T18:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:58Z" name: isvc-sklearn-graph-2.18d5e233ab355807 namespace: kserve-ci-e2e-test resourceVersion: "16106" uid: 4e7d003c-a49d-4e40-9cbd-2cc3919b255c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:57:05.270033Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-2k99h to ip-10-0-134-125.ec2.internal metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:57:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e2194e94416a namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: 758069b0-2999-4932-9644-365d42ba057c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:57:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e219710705c1 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: c17a9a49-a7f3-4a98-946b-2f467be93976 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:a69470ab71f96748343c847bfecfdfb03e198759f46531295e483d003e69266a" already present on machine metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e21972a553fe namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: dc7146d9-9bb8-4e9a-a223-3f8b89ec9f0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e21975eafb89 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 7b7c9511-0e11-4d1d-add0-5418131388ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:05Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e219766639e5 namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: 78c3de61-a1d0-48c9-98e8-ef37bac9f53e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:57:10Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-16T18:57:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e21a712bb1b2 namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: d427c147-b947-4128-a7c6-770feaa02f19 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:57:38Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 28.594s (28.594s including waiting). Image size: 1332365570 bytes.' metadata: creationTimestamp: "2026-09-16T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e221198a9d17 namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 3c3ecfc8-768c-46a7-95d7-7c2d85b75ccc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:57:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e2211f97dbe1 namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: 083edf8a-bd6e-4b62-b46d-3523b7887815 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:57:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e2212056ac5f namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: 88837121-38a7-453a-bad7-d4004ece9d9b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:57:38Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-16T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e221206af278 namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: b5bc3361-f7dd-4087-92ca-2195bf04e355 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:57:41Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.385s (2.385s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-16T18:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e221ae9e1f4c namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: ed44f955-5499-42d5-81ce-87054e538bd1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:57:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e221b2e81819 namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: 21f82514-634c-40b3-8599-fcdca93fa4f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:57:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e221b3933588 namespace: kserve-ci-e2e-test resourceVersion: "15240" uid: 5ba3045f-7562-460b-bd04-dbcc6a1309dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-16T18:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:59:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e221b69ae36b namespace: kserve-ci-e2e-test resourceVersion: "16722" uid: 7f41de1e-6d25-467e-b551-397d9e2764c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e23c2d33d22e namespace: kserve-ci-e2e-test resourceVersion: "16572" uid: c60561e1-4ce7-40b2-a1fb-40e1aa355ee3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e23c2d346440 namespace: kserve-ci-e2e-test resourceVersion: "16573" uid: 8d8d03d5-5d22-4a02-b9cd-1994b337394d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2k99h namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: b8ce26d9-bfc1-4555-bc6e-47b6fc0e234c kind: Event lastTimestamp: "2026-09-16T18:59:38Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-2k99h.18d5e23cf5004894 namespace: kserve-ci-e2e-test resourceVersion: "16720" uid: 0b9ee788-2dc2-4612-ab13-c77d63c79196 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-125.ec2.internal source: component: kubelet host: ip-10-0-134-125.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14801" uid: 218a1a8e-c500-44b6-b1e6-8ac907553172 kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-2k99h' metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:05Z" name: isvc-xgboost-graph-predictor-555fc69d56.18d5e2194df6d76d namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 60292893-de7f-482d-8538-8b2efa6c5548 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 17a3ac6e-e1a7-4c82-93e7-255c31c12d1f kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:05Z" name: isvc-xgboost-graph-predictor.18d5e21946c38c21 namespace: kserve-ci-e2e-test resourceVersion: "14802" uid: a8fc9c09-0b4b-4507-8fa8-0650748046a9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:57:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 73f96097-1bbc-4e00-a616-2f1ea636874c kind: Event lastTimestamp: "2026-09-16T18:57:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:50Z" name: isvc-xgboost-graph-predictor.18d5e21cc5597064 namespace: kserve-ci-e2e-test resourceVersion: "15326" uid: c8e6ac39-18e1-4efa-a5f3-ffe1a4d88d96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:57:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 73f96097-1bbc-4e00-a616-2f1ea636874c kind: Event lastTimestamp: "2026-09-16T18:57:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:50Z" name: isvc-xgboost-graph-predictor.18d5e21cc5cc45aa namespace: kserve-ci-e2e-test resourceVersion: "15327" uid: 81557d7e-ce79-4fd5-8645-a3b09751caab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:58:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 73f96097-1bbc-4e00-a616-2f1ea636874c kind: Event lastTimestamp: "2026-09-16T18:59:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:58:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:20Z" name: isvc-xgboost-graph-predictor.18d5e22741e5b981 namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: 83561468-8e7e-4291-972d-71bb69f6c220 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:58:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15044" uid: 73f96097-1bbc-4e00-a616-2f1ea636874c kind: Event lastTimestamp: "2026-09-16T18:59:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:58:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:20Z" name: isvc-xgboost-graph-predictor.18d5e227424fdf68 namespace: kserve-ci-e2e-test resourceVersion: "16338" uid: f9b0b026-a09d-4fa7-9f49-20d5dbc745c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: f1c66e5e-08f6-4a4a-8760-c72a75f60143 kind: Event lastTimestamp: "2026-09-16T18:58:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:48Z" name: isvc-xgboost-graph.18d5e21946e8559f namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 99467c1a-2b1f-4b1c-b8e5-0cbb2e5b6b25 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: f1c66e5e-08f6-4a4a-8760-c72a75f60143 kind: Event lastTimestamp: "2026-09-16T18:58:48Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-09-16T18:58:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:48Z" name: isvc-xgboost-graph.18d5e23156eef0e7 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: a2f7cf8a-7aeb-4d16-8800-a6da6d66ef4e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:59:24.758172Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-6cd7f458db-8tkvv namespace: kserve-ci-e2e-test resourceVersion: "16400" uid: 958bc847-30e5-4e63-8bbe-71ec1d16dd90 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-6cd7f458db-8tkvv to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T18:59:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:59:24Z" name: model-chainer-6cd7f458db-8tkvv.18d5e239c8b8659c namespace: kserve-ci-e2e-test resourceVersion: "16405" uid: 3dcb02c6-5ea6-4561-8d72-167915d6d9a6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:25Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-6cd7f458db-8tkvv namespace: kserve-ci-e2e-test resourceVersion: "16411" uid: 958bc847-30e5-4e63-8bbe-71ec1d16dd90 kind: Event lastTimestamp: "2026-09-16T18:59:25Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:59:25Z" name: model-chainer-6cd7f458db-8tkvv.18d5e239eaf376e6 namespace: kserve-ci-e2e-test resourceVersion: "16423" uid: afc3fd85-9bc9-486b-905a-40e16678de2d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6cd7f458db-8tkvv namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: 958bc847-30e5-4e63-8bbe-71ec1d16dd90 kind: Event lastTimestamp: "2026-09-16T18:59:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:83a04242aef3bc251b4da1ba3ce544c38d7df9880cbb933157aae8432a36f471" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:25Z" name: model-chainer-6cd7f458db-8tkvv.18d5e239ec64a7a4 namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: 544c373d-9bf6-482c-b56d-b858ec739258 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6cd7f458db-8tkvv namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: 958bc847-30e5-4e63-8bbe-71ec1d16dd90 kind: Event lastTimestamp: "2026-09-16T18:59:25Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-09-16T18:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:25Z" name: model-chainer-6cd7f458db-8tkvv.18d5e239ef16a3d8 namespace: kserve-ci-e2e-test resourceVersion: "16426" uid: 1f2c00b5-977c-401f-b8fa-c9597e484a50 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6cd7f458db-8tkvv namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: 958bc847-30e5-4e63-8bbe-71ec1d16dd90 kind: Event lastTimestamp: "2026-09-16T18:59:25Z" message: Started container model-chainer metadata: creationTimestamp: "2026-09-16T18:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:25Z" name: model-chainer-6cd7f458db-8tkvv.18d5e239ef82bbcb namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: a9b7ccb3-1f2b-4965-87f2-667aa0904a85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6cd7f458db-8tkvv namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: 958bc847-30e5-4e63-8bbe-71ec1d16dd90 kind: Event lastTimestamp: "2026-09-16T18:59:34Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-09-16T18:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:34Z" name: model-chainer-6cd7f458db-8tkvv.18d5e23c23354ac8 namespace: kserve-ci-e2e-test resourceVersion: "16514" uid: 56425050-afc0-47d3-a23a-788e7b3cc861 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:59:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6cd7f458db-8tkvv namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: 958bc847-30e5-4e63-8bbe-71ec1d16dd90 kind: Event lastTimestamp: "2026-09-16T19:00:01Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T18:59:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:01Z" name: model-chainer-6cd7f458db-8tkvv.18d5e23c9307fdce namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: bbfe0ce4-4da1-41cf-9799-3d70d93301e7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-6cd7f458db namespace: kserve-ci-e2e-test resourceVersion: "16397" uid: 1de0ce21-994c-4bf1-bd6f-b0a89515ee56 kind: Event lastTimestamp: "2026-09-16T18:59:24Z" message: 'Created pod: model-chainer-6cd7f458db-8tkvv' metadata: creationTimestamp: "2026-09-16T18:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:24Z" name: model-chainer-6cd7f458db.18d5e239c859701f namespace: kserve-ci-e2e-test resourceVersion: "16402" uid: ec4d16e2-f282-4c31-81cb-fe79291c9388 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: 3d066bf9-286e-43ac-85eb-1710668f46e8 kind: Event lastTimestamp: "2026-09-16T18:59:24Z" message: Scaled up replica set model-chainer-6cd7f458db from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:24Z" name: model-chainer.18d5e239c78afe5d namespace: kserve-ci-e2e-test resourceVersion: "16398" uid: 0f953e61-497f-4b5e-bf06-50582d92bfca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 12c92c4a-e7e9-42c2-92ee-2fd85b880188 kind: Event lastTimestamp: "2026-09-16T18:59:24Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-09-16T18:59:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:59:24Z" name: model-chainer.18d5e239cd2ff2db namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 1dad1faa-aa8b-4a6f-b19d-720bf6b620df reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:11:00.301555Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-53a10-756f44fbbb-hpvpp namespace: kserve-ci-e2e-test resourceVersion: "22435" uid: d6c2b4d2-499b-4975-901d-7a6f28e71ab0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-53a10-756f44fbbb-hpvpp to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:11:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:11:00Z" name: sequence-graph-53a10-756f44fbbb-hpvpp.18d5e2dbba561bcf namespace: kserve-ci-e2e-test resourceVersion: "22442" uid: beed02ce-c524-4d29-a8cb-08416bb2f679 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:11:00Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-53a10-756f44fbbb-hpvpp namespace: kserve-ci-e2e-test resourceVersion: "22437" uid: d6c2b4d2-499b-4975-901d-7a6f28e71ab0 kind: Event lastTimestamp: "2026-09-16T19:11:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-53a10-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:00Z" name: sequence-graph-53a10-756f44fbbb-hpvpp.18d5e2dbc154d13c namespace: kserve-ci-e2e-test resourceVersion: "22461" uid: c7f817cb-b584-494f-9b36-f7c662765add reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:02Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-53a10-756f44fbbb-hpvpp namespace: kserve-ci-e2e-test resourceVersion: "22446" uid: d6c2b4d2-499b-4975-901d-7a6f28e71ab0 kind: Event lastTimestamp: "2026-09-16T19:11:02Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:11:02Z" name: sequence-graph-53a10-756f44fbbb-hpvpp.18d5e2dc35a3001b namespace: kserve-ci-e2e-test resourceVersion: "22479" uid: 00260dd8-4cd4-47d7-b7ed-883e5b93c030 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-53a10} kind: Pod name: sequence-graph-53a10-756f44fbbb-hpvpp namespace: kserve-ci-e2e-test resourceVersion: "22437" uid: d6c2b4d2-499b-4975-901d-7a6f28e71ab0 kind: Event lastTimestamp: "2026-09-16T19:11:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:83a04242aef3bc251b4da1ba3ce544c38d7df9880cbb933157aae8432a36f471" already present on machine metadata: creationTimestamp: "2026-09-16T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:02Z" name: sequence-graph-53a10-756f44fbbb-hpvpp.18d5e2dc36f44a23 namespace: kserve-ci-e2e-test resourceVersion: "22481" uid: 96b593a0-0847-463a-968f-6a455649694b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-53a10} kind: Pod name: sequence-graph-53a10-756f44fbbb-hpvpp namespace: kserve-ci-e2e-test resourceVersion: "22437" uid: d6c2b4d2-499b-4975-901d-7a6f28e71ab0 kind: Event lastTimestamp: "2026-09-16T19:11:02Z" message: 'Created container: sequence-graph-53a10' metadata: creationTimestamp: "2026-09-16T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:02Z" name: sequence-graph-53a10-756f44fbbb-hpvpp.18d5e2dc39aff68f namespace: kserve-ci-e2e-test resourceVersion: "22482" uid: 28dd587b-1b98-4984-b18f-8c70d251d722 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-53a10} kind: Pod name: sequence-graph-53a10-756f44fbbb-hpvpp namespace: kserve-ci-e2e-test resourceVersion: "22437" uid: d6c2b4d2-499b-4975-901d-7a6f28e71ab0 kind: Event lastTimestamp: "2026-09-16T19:11:02Z" message: Started container sequence-graph-53a10 metadata: creationTimestamp: "2026-09-16T19:11:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:11:02Z" name: sequence-graph-53a10-756f44fbbb-hpvpp.18d5e2dc3a1c9f9a namespace: kserve-ci-e2e-test resourceVersion: "22483" uid: 0e476077-7e53-475f-9bb3-46c917b06d52 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-53a10} kind: Pod name: sequence-graph-53a10-756f44fbbb-hpvpp namespace: kserve-ci-e2e-test resourceVersion: "22437" uid: d6c2b4d2-499b-4975-901d-7a6f28e71ab0 kind: Event lastTimestamp: "2026-09-16T19:19:14Z" message: Stopping container sequence-graph-53a10 metadata: creationTimestamp: "2026-09-16T19:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:14Z" name: sequence-graph-53a10-756f44fbbb-hpvpp.18d5e34ee75b8ec5 namespace: kserve-ci-e2e-test resourceVersion: "26352" uid: 44ce0e80-1714-4576-8f8c-c80e5fdeac09 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:19:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-53a10} kind: Pod name: sequence-graph-53a10-756f44fbbb-hpvpp namespace: kserve-ci-e2e-test resourceVersion: "22437" uid: d6c2b4d2-499b-4975-901d-7a6f28e71ab0 kind: Event lastTimestamp: "2026-09-16T19:19:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:19:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:44Z" name: sequence-graph-53a10-756f44fbbb-hpvpp.18d5e34feb26732a namespace: kserve-ci-e2e-test resourceVersion: "26918" uid: 71b46d6c-9de8-4ef3-aa51-2597aa509813 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-53a10-756f44fbbb namespace: kserve-ci-e2e-test resourceVersion: "22432" uid: 09db5ad7-90eb-4da6-87da-cbedc00f1e55 kind: Event lastTimestamp: "2026-09-16T19:11:00Z" message: 'Created pod: sequence-graph-53a10-756f44fbbb-hpvpp' metadata: creationTimestamp: "2026-09-16T19:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:11:00Z" name: sequence-graph-53a10-756f44fbbb.18d5e2dbb9d6c58a namespace: kserve-ci-e2e-test resourceVersion: "22436" uid: 32042ca5-2c27-4d05-98fe-78cab8ba705e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-53a10 namespace: kserve-ci-e2e-test resourceVersion: "22430" uid: 6b943980-5bd6-4297-a2ea-463099564a67 kind: Event lastTimestamp: "2026-09-16T19:11:00Z" message: Scaled up replica set sequence-graph-53a10-756f44fbbb from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:11:00Z" name: sequence-graph-53a10.18d5e2dbb8f15c72 namespace: kserve-ci-e2e-test resourceVersion: "22434" uid: 29e8b13c-77ff-49d3-9a19-4bf56b8dd778 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-53a10 namespace: kserve-ci-e2e-test resourceVersion: "22451" uid: 245d5ab4-c28e-4e14-87f3-5c17a91d661a kind: Event lastTimestamp: "2026-09-16T19:11:00Z" message: InferenceGraph [sequence-graph-53a10] is Ready metadata: creationTimestamp: "2026-09-16T19:11:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:11:00Z" name: sequence-graph-53a10.18d5e2dbbf8821ef namespace: kserve-ci-e2e-test resourceVersion: "22453" uid: 86c16afb-7e83-4200-bc36-1e8211ab3e09 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-53a10 namespace: kserve-ci-e2e-test resourceVersion: "22431" uid: 2d167410-dbc7-4993-a9cc-47bfa342004e kind: Event lastTimestamp: "2026-09-16T19:11:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:11:15Z" name: sequence-graph-53a10.18d5e2df37a1e750 namespace: kserve-ci-e2e-test resourceVersion: "22578" uid: 4ac0eec7-e592-46ab-aacd-377d610b3d2c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-53a10 namespace: kserve-ci-e2e-test resourceVersion: "22431" uid: 2d167410-dbc7-4993-a9cc-47bfa342004e kind: Event lastTimestamp: "2026-09-16T19:11:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:11:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:11:15Z" name: sequence-graph-53a10.18d5e2df3803c2e7 namespace: kserve-ci-e2e-test resourceVersion: "22579" uid: 2e69efc9-bef7-46e1-b9fb-82379d15944f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-53a10 namespace: kserve-ci-e2e-test resourceVersion: "22577" uid: 2d167410-dbc7-4993-a9cc-47bfa342004e kind: Event lastTimestamp: "2026-09-16T19:11:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:11:30Z" name: sequence-graph-53a10.18d5e2e2b7cc4b76 namespace: kserve-ci-e2e-test resourceVersion: "22683" uid: 1b16a805-c3e8-4456-8c7b-ac3d5c0aa28c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:11:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-53a10 namespace: kserve-ci-e2e-test resourceVersion: "22577" uid: 2d167410-dbc7-4993-a9cc-47bfa342004e kind: Event lastTimestamp: "2026-09-16T19:11:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:11:30Z" name: sequence-graph-53a10.18d5e2e2b8620de0 namespace: kserve-ci-e2e-test resourceVersion: "22687" uid: 8b7817a1-e700-4501-9258-69bf92ea7e2b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:09:49.947918Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9450b-587fc5878c-n2qk6 namespace: kserve-ci-e2e-test resourceVersion: "21659" uid: 9139b83b-964b-4bf2-be86-03fc1ee74e7e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-9450b-587fc5878c-n2qk6 to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:09:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:09:49Z" name: sequence-graph-9450b-587fc5878c-n2qk6.18d5e2cb58eec64b namespace: kserve-ci-e2e-test resourceVersion: "21664" uid: d8d8cc9e-e28e-4711-99e8-fea1d78d6d4f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:50Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9450b-587fc5878c-n2qk6 namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 9139b83b-964b-4bf2-be86-03fc1ee74e7e kind: Event lastTimestamp: "2026-09-16T19:09:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-9450b-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:50Z" name: sequence-graph-9450b-587fc5878c-n2qk6.18d5e2cb63c39b79 namespace: kserve-ci-e2e-test resourceVersion: "21680" uid: cb5a59b2-9e2a-47d7-aa1f-709a310ba93b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:51Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-9450b-587fc5878c-n2qk6 namespace: kserve-ci-e2e-test resourceVersion: "21669" uid: 9139b83b-964b-4bf2-be86-03fc1ee74e7e kind: Event lastTimestamp: "2026-09-16T19:09:51Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:09:51Z" name: sequence-graph-9450b-587fc5878c-n2qk6.18d5e2cb9eea7ce2 namespace: kserve-ci-e2e-test resourceVersion: "21690" uid: a62fe5f2-2839-4c35-a92a-91260a031c40 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9450b} kind: Pod name: sequence-graph-9450b-587fc5878c-n2qk6 namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 9139b83b-964b-4bf2-be86-03fc1ee74e7e kind: Event lastTimestamp: "2026-09-16T19:09:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:83a04242aef3bc251b4da1ba3ce544c38d7df9880cbb933157aae8432a36f471" already present on machine metadata: creationTimestamp: "2026-09-16T19:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:51Z" name: sequence-graph-9450b-587fc5878c-n2qk6.18d5e2cba044ed33 namespace: kserve-ci-e2e-test resourceVersion: "21692" uid: 0f8f8a0e-7ff9-48a5-b970-8d242dc4614d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9450b} kind: Pod name: sequence-graph-9450b-587fc5878c-n2qk6 namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 9139b83b-964b-4bf2-be86-03fc1ee74e7e kind: Event lastTimestamp: "2026-09-16T19:09:51Z" message: 'Created container: sequence-graph-9450b' metadata: creationTimestamp: "2026-09-16T19:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:51Z" name: sequence-graph-9450b-587fc5878c-n2qk6.18d5e2cba32aacf7 namespace: kserve-ci-e2e-test resourceVersion: "21694" uid: a06c1dcb-c23a-4c45-82bc-b999ede09054 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9450b} kind: Pod name: sequence-graph-9450b-587fc5878c-n2qk6 namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 9139b83b-964b-4bf2-be86-03fc1ee74e7e kind: Event lastTimestamp: "2026-09-16T19:09:51Z" message: Started container sequence-graph-9450b metadata: creationTimestamp: "2026-09-16T19:09:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:51Z" name: sequence-graph-9450b-587fc5878c-n2qk6.18d5e2cba3941968 namespace: kserve-ci-e2e-test resourceVersion: "21695" uid: 5bcf0687-9e3c-4415-9fe7-46d75b229447 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9450b} kind: Pod name: sequence-graph-9450b-587fc5878c-n2qk6 namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 9139b83b-964b-4bf2-be86-03fc1ee74e7e kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: Stopping container sequence-graph-9450b metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:00Z" name: sequence-graph-9450b-587fc5878c-n2qk6.18d5e2cdb2a7604f namespace: kserve-ci-e2e-test resourceVersion: "21775" uid: 34d7ad07-577e-4eb5-85d2-a5436f52b53a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:10:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-9450b} kind: Pod name: sequence-graph-9450b-587fc5878c-n2qk6 namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: 9139b83b-964b-4bf2-be86-03fc1ee74e7e kind: Event lastTimestamp: "2026-09-16T19:10:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:28Z" name: sequence-graph-9450b-587fc5878c-n2qk6.18d5e2ce66b58880 namespace: kserve-ci-e2e-test resourceVersion: "22167" uid: 50892741-a38d-4205-9533-8a1f4ed88a82 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-9450b-587fc5878c namespace: kserve-ci-e2e-test resourceVersion: "21656" uid: ea77d4de-38bf-40d1-b182-4f9e15317e9c kind: Event lastTimestamp: "2026-09-16T19:09:49Z" message: 'Created pod: sequence-graph-9450b-587fc5878c-n2qk6' metadata: creationTimestamp: "2026-09-16T19:09:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:49Z" name: sequence-graph-9450b-587fc5878c.18d5e2cb5879ddd3 namespace: kserve-ci-e2e-test resourceVersion: "21660" uid: f915209f-f4ec-43fb-a7f6-012268879f02 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-9450b namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: b7094dad-6b17-41cc-845d-4f0d83730c5e kind: Event lastTimestamp: "2026-09-16T19:09:49Z" message: Scaled up replica set sequence-graph-9450b-587fc5878c from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:09:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:49Z" name: sequence-graph-9450b.18d5e2cb57965a18 namespace: kserve-ci-e2e-test resourceVersion: "21658" uid: 13c54f94-2af8-4484-b13b-a73a669c56c7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-9450b namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: ddd360e0-fc79-40f0-a5e0-e318480baf52 kind: Event lastTimestamp: "2026-09-16T19:09:50Z" message: InferenceGraph [sequence-graph-9450b] is Ready metadata: creationTimestamp: "2026-09-16T19:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:09:50Z" name: sequence-graph-9450b.18d5e2cb5cea41cf namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: 1c6a7f9d-0696-48ab-ae93-d6e1c332efee reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:00:35.088585Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a895e-569fdff559-p2sc5 namespace: kserve-ci-e2e-test resourceVersion: "17208" uid: da3911f4-2ade-4427-92f0-7f041c973e61 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a895e-569fdff559-p2sc5 to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:00:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:00:35Z" name: sequence-graph-a895e-569fdff559-p2sc5.18d5e24a28bd44e1 namespace: kserve-ci-e2e-test resourceVersion: "17213" uid: 98ceae8c-5541-4fbf-b906-196c0ed1b091 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:35Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a895e-569fdff559-p2sc5 namespace: kserve-ci-e2e-test resourceVersion: "17218" uid: da3911f4-2ade-4427-92f0-7f041c973e61 kind: Event lastTimestamp: "2026-09-16T19:00:35Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:00:35Z" name: sequence-graph-a895e-569fdff559-p2sc5.18d5e24a4a97ad7a namespace: kserve-ci-e2e-test resourceVersion: "17235" uid: 7c6203e3-e6da-49bd-a00d-bbc867d982b5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a895e} kind: Pod name: sequence-graph-a895e-569fdff559-p2sc5 namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: da3911f4-2ade-4427-92f0-7f041c973e61 kind: Event lastTimestamp: "2026-09-16T19:00:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:83a04242aef3bc251b4da1ba3ce544c38d7df9880cbb933157aae8432a36f471" already present on machine metadata: creationTimestamp: "2026-09-16T19:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:35Z" name: sequence-graph-a895e-569fdff559-p2sc5.18d5e24a4c969f43 namespace: kserve-ci-e2e-test resourceVersion: "17237" uid: 8142b961-f355-4039-b21b-ad0532ef517f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a895e} kind: Pod name: sequence-graph-a895e-569fdff559-p2sc5 namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: da3911f4-2ade-4427-92f0-7f041c973e61 kind: Event lastTimestamp: "2026-09-16T19:00:35Z" message: 'Created container: sequence-graph-a895e' metadata: creationTimestamp: "2026-09-16T19:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:35Z" name: sequence-graph-a895e-569fdff559-p2sc5.18d5e24a4fe2981a namespace: kserve-ci-e2e-test resourceVersion: "17238" uid: b5e4c394-e04e-477f-8567-725f8dbedfd1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a895e} kind: Pod name: sequence-graph-a895e-569fdff559-p2sc5 namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: da3911f4-2ade-4427-92f0-7f041c973e61 kind: Event lastTimestamp: "2026-09-16T19:00:35Z" message: Started container sequence-graph-a895e metadata: creationTimestamp: "2026-09-16T19:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:35Z" name: sequence-graph-a895e-569fdff559-p2sc5.18d5e24a5050e143 namespace: kserve-ci-e2e-test resourceVersion: "17239" uid: 3ce608e1-7e1a-44eb-889e-686dafa25622 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a895e} kind: Pod name: sequence-graph-a895e-569fdff559-p2sc5 namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: da3911f4-2ade-4427-92f0-7f041c973e61 kind: Event lastTimestamp: "2026-09-16T19:08:49Z" message: Stopping container sequence-graph-a895e metadata: creationTimestamp: "2026-09-16T19:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:49Z" name: sequence-graph-a895e-569fdff559-p2sc5.18d5e2bd54378cf8 namespace: kserve-ci-e2e-test resourceVersion: "20808" uid: 26bcc623-0790-4532-900c-60f40b18f583 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:08:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a895e} kind: Pod name: sequence-graph-a895e-569fdff559-p2sc5 namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: da3911f4-2ade-4427-92f0-7f041c973e61 kind: Event lastTimestamp: "2026-09-16T19:09:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:17Z" name: sequence-graph-a895e-569fdff559-p2sc5.18d5e2bddace73fd namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: d6e942fb-2476-4353-bd3d-b9ce4e59f7fd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a895e-569fdff559 namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: 9ae39d10-652f-4d50-99d0-b74505dde3dd kind: Event lastTimestamp: "2026-09-16T19:00:35Z" message: 'Created pod: sequence-graph-a895e-569fdff559-p2sc5' metadata: creationTimestamp: "2026-09-16T19:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:35Z" name: sequence-graph-a895e-569fdff559.18d5e24a27f0810e namespace: kserve-ci-e2e-test resourceVersion: "17209" uid: 651b41c6-298d-4722-a013-6be0bc1e786c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a895e namespace: kserve-ci-e2e-test resourceVersion: "17201" uid: 14810531-05d4-4cac-b646-43e1cf167a1e kind: Event lastTimestamp: "2026-09-16T19:00:35Z" message: Scaled up replica set sequence-graph-a895e-569fdff559 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:35Z" name: sequence-graph-a895e.18d5e24a26d79813 namespace: kserve-ci-e2e-test resourceVersion: "17206" uid: b3a686db-7e95-4c4d-83fc-cf7c4c9fb3dc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a895e namespace: kserve-ci-e2e-test resourceVersion: "17221" uid: 5aaeb59a-e8b6-4cb6-b61a-aaa629b3d016 kind: Event lastTimestamp: "2026-09-16T19:00:35Z" message: InferenceGraph [sequence-graph-a895e] is Ready metadata: creationTimestamp: "2026-09-16T19:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:00:35Z" name: sequence-graph-a895e.18d5e24a2e49118b namespace: kserve-ci-e2e-test resourceVersion: "17223" uid: b31eab39-e69b-480e-b505-2e529aa51bf1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:00:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a895e namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: d48dde70-f55d-49b0-886d-e33f7184ec13 kind: Event lastTimestamp: "2026-09-16T19:01:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:01:20Z" name: sequence-graph-a895e.18d5e24da5a0a5a4 namespace: kserve-ci-e2e-test resourceVersion: "17558" uid: 147ffbc7-a296-458b-b29f-25f744b80dbf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:00:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a895e namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: d48dde70-f55d-49b0-886d-e33f7184ec13 kind: Event lastTimestamp: "2026-09-16T19:01:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:01:20Z" name: sequence-graph-a895e.18d5e24da61edaee namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: 1144fc2a-49f1-494b-a46d-8e59e66deea0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:20:25.745606Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-17a4d-6c9ff65448-jm9m7 namespace: kserve-ci-e2e-test resourceVersion: "27323" uid: 7f1a25f0-8e32-407c-83ed-a6112d4c7a6d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-17a4d-6c9ff65448-jm9m7 to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:20:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:20:25Z" name: splitter-graph-17a4d-6c9ff65448-jm9m7.18d5e35f616d9880 namespace: kserve-ci-e2e-test resourceVersion: "27329" uid: e5ed10ec-c1b6-4db9-b105-45cc373da5af reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:26Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-17a4d-6c9ff65448-jm9m7 namespace: kserve-ci-e2e-test resourceVersion: "27334" uid: 7f1a25f0-8e32-407c-83ed-a6112d4c7a6d kind: Event lastTimestamp: "2026-09-16T19:20:26Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:20:26Z" name: splitter-graph-17a4d-6c9ff65448-jm9m7.18d5e35f836e9836 namespace: kserve-ci-e2e-test resourceVersion: "27346" uid: e3417def-014c-49ba-a60f-e0cfefe0d82a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-17a4d} kind: Pod name: splitter-graph-17a4d-6c9ff65448-jm9m7 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f1a25f0-8e32-407c-83ed-a6112d4c7a6d kind: Event lastTimestamp: "2026-09-16T19:20:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:83a04242aef3bc251b4da1ba3ce544c38d7df9880cbb933157aae8432a36f471" already present on machine metadata: creationTimestamp: "2026-09-16T19:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:20:26Z" name: splitter-graph-17a4d-6c9ff65448-jm9m7.18d5e35f84d3b445 namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: b5f43169-38ea-4096-a97f-6855e03e368e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-17a4d} kind: Pod name: splitter-graph-17a4d-6c9ff65448-jm9m7 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f1a25f0-8e32-407c-83ed-a6112d4c7a6d kind: Event lastTimestamp: "2026-09-16T19:20:26Z" message: 'Created container: splitter-graph-17a4d' metadata: creationTimestamp: "2026-09-16T19:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:20:26Z" name: splitter-graph-17a4d-6c9ff65448-jm9m7.18d5e35f879b29d3 namespace: kserve-ci-e2e-test resourceVersion: "27352" uid: 3233ae20-1968-4db8-8aa4-2f2e58bf3ef1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-17a4d} kind: Pod name: splitter-graph-17a4d-6c9ff65448-jm9m7 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f1a25f0-8e32-407c-83ed-a6112d4c7a6d kind: Event lastTimestamp: "2026-09-16T19:20:26Z" message: Started container splitter-graph-17a4d metadata: creationTimestamp: "2026-09-16T19:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:20:26Z" name: splitter-graph-17a4d-6c9ff65448-jm9m7.18d5e35f88060d14 namespace: kserve-ci-e2e-test resourceVersion: "27353" uid: c5da7a9a-ce5e-4a66-bcfe-2e479342b0a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-17a4d} kind: Pod name: splitter-graph-17a4d-6c9ff65448-jm9m7 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f1a25f0-8e32-407c-83ed-a6112d4c7a6d kind: Event lastTimestamp: "2026-09-16T19:28:40Z" message: Stopping container splitter-graph-17a4d metadata: creationTimestamp: "2026-09-16T19:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:28:40Z" name: splitter-graph-17a4d-6c9ff65448-jm9m7.18d5e3d28415381d namespace: kserve-ci-e2e-test resourceVersion: "30776" uid: 8cf09a86-1e15-497d-9b09-8746859892b9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-17a4d} kind: Pod name: splitter-graph-17a4d-6c9ff65448-jm9m7 namespace: kserve-ci-e2e-test resourceVersion: "27325" uid: 7f1a25f0-8e32-407c-83ed-a6112d4c7a6d kind: Event lastTimestamp: "2026-09-16T19:29:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:29:07Z" name: splitter-graph-17a4d-6c9ff65448-jm9m7.18d5e3d3061f326d namespace: kserve-ci-e2e-test resourceVersion: "31009" uid: 092bb291-a95d-4c34-a626-fa4e6a77a863 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-17a4d-6c9ff65448 namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: 0bd8299e-f099-4eac-b8df-8025e784a1e5 kind: Event lastTimestamp: "2026-09-16T19:20:25Z" message: 'Created pod: splitter-graph-17a4d-6c9ff65448-jm9m7' metadata: creationTimestamp: "2026-09-16T19:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:25Z" name: splitter-graph-17a4d-6c9ff65448.18d5e35f60b23c31 namespace: kserve-ci-e2e-test resourceVersion: "27324" uid: 0fb91908-fa11-4390-999b-dcdb0dc49a0e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-17a4d namespace: kserve-ci-e2e-test resourceVersion: "27318" uid: 2654e72a-06c8-478a-b2c3-f59b77eb11bc kind: Event lastTimestamp: "2026-09-16T19:20:25Z" message: Scaled up replica set splitter-graph-17a4d-6c9ff65448 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:25Z" name: splitter-graph-17a4d.18d5e35f5fda0dfd namespace: kserve-ci-e2e-test resourceVersion: "27321" uid: ae7e266a-543a-4689-992b-e38fac2c24f8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-17a4d namespace: kserve-ci-e2e-test resourceVersion: "27339" uid: 7cf2b47a-0426-45bc-b445-d4875816cb1b kind: Event lastTimestamp: "2026-09-16T19:20:25Z" message: InferenceGraph [splitter-graph-17a4d] is Ready metadata: creationTimestamp: "2026-09-16T19:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:20:25Z" name: splitter-graph-17a4d.18d5e35f67c9adf8 namespace: kserve-ci-e2e-test resourceVersion: "27342" uid: 8667d973-82d7-4ba0-a1cf-f30aea8a15d6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:20:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-17a4d namespace: kserve-ci-e2e-test resourceVersion: "27319" uid: caebc1ac-0807-4538-b289-3db8d7571961 kind: Event lastTimestamp: "2026-09-16T19:20:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:55Z" name: splitter-graph-17a4d.18d5e362def278ba namespace: kserve-ci-e2e-test resourceVersion: "27579" uid: 211a7267-8e3c-492b-8f5d-5cc6738d7f4f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:20:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-17a4d namespace: kserve-ci-e2e-test resourceVersion: "27319" uid: caebc1ac-0807-4538-b289-3db8d7571961 kind: Event lastTimestamp: "2026-09-16T19:20:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:55Z" name: splitter-graph-17a4d.18d5e362df79893d namespace: kserve-ci-e2e-test resourceVersion: "27580" uid: 90605b35-ec71-44ff-9c32-9954b3545a43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:19:15.433987Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-3bba6-879cd4d78-6rx4b namespace: kserve-ci-e2e-test resourceVersion: "26456" uid: 7154dd1e-7dd6-4a7d-b671-6ed9d1776058 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-3bba6-879cd4d78-6rx4b to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:19:15Z" name: splitter-graph-3bba6-879cd4d78-6rx4b.18d5e34f02878b57 namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 474a7c2c-1409-4ec8-857e-cc2efedd852f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-3bba6-879cd4d78-6rx4b namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 7154dd1e-7dd6-4a7d-b671-6ed9d1776058 kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-3bba6-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:15Z" name: splitter-graph-3bba6-879cd4d78-6rx4b.18d5e34f0be6f166 namespace: kserve-ci-e2e-test resourceVersion: "26483" uid: fa6b4e3c-7f39-4ee9-8da4-fb8e93d392da reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-3bba6-879cd4d78-6rx4b namespace: kserve-ci-e2e-test resourceVersion: "26467" uid: 7154dd1e-7dd6-4a7d-b671-6ed9d1776058 kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:19:16Z" name: splitter-graph-3bba6-879cd4d78-6rx4b.18d5e34f4824281d namespace: kserve-ci-e2e-test resourceVersion: "26507" uid: 2ba8e751-1ff8-4fcb-8677-707cca1f7af9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3bba6} kind: Pod name: splitter-graph-3bba6-879cd4d78-6rx4b namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 7154dd1e-7dd6-4a7d-b671-6ed9d1776058 kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:83a04242aef3bc251b4da1ba3ce544c38d7df9880cbb933157aae8432a36f471" already present on machine metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: splitter-graph-3bba6-879cd4d78-6rx4b.18d5e34f495242b8 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: e0278a3f-56ef-47b8-b16d-69c0ceeb64b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3bba6} kind: Pod name: splitter-graph-3bba6-879cd4d78-6rx4b namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 7154dd1e-7dd6-4a7d-b671-6ed9d1776058 kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: 'Created container: splitter-graph-3bba6' metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: splitter-graph-3bba6-879cd4d78-6rx4b.18d5e34f4c0c3957 namespace: kserve-ci-e2e-test resourceVersion: "26513" uid: f4cc0b28-b091-4965-9785-b6f31958eacd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3bba6} kind: Pod name: splitter-graph-3bba6-879cd4d78-6rx4b namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 7154dd1e-7dd6-4a7d-b671-6ed9d1776058 kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: Started container splitter-graph-3bba6 metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: splitter-graph-3bba6-879cd4d78-6rx4b.18d5e34f4c78d12b namespace: kserve-ci-e2e-test resourceVersion: "26514" uid: 676c5b16-b5a7-43d0-9df7-a831030432f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3bba6} kind: Pod name: splitter-graph-3bba6-879cd4d78-6rx4b namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 7154dd1e-7dd6-4a7d-b671-6ed9d1776058 kind: Event lastTimestamp: "2026-09-16T19:19:25Z" message: Stopping container splitter-graph-3bba6 metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:25Z" name: splitter-graph-3bba6-879cd4d78-6rx4b.18d5e3515a02ac54 namespace: kserve-ci-e2e-test resourceVersion: "26628" uid: 58094fe5-1ebd-43b9-b0b4-23aa62a113db reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:19:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-3bba6} kind: Pod name: splitter-graph-3bba6-879cd4d78-6rx4b namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: 7154dd1e-7dd6-4a7d-b671-6ed9d1776058 kind: Event lastTimestamp: "2026-09-16T19:19:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:53Z" name: splitter-graph-3bba6-879cd4d78-6rx4b.18d5e351f95e7663 namespace: kserve-ci-e2e-test resourceVersion: "26986" uid: 36210709-0a4d-48b0-b78f-036b02babf71 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-3bba6-879cd4d78 namespace: kserve-ci-e2e-test resourceVersion: "26453" uid: 088c568a-4dd5-4e5a-98a2-73e83be2f248 kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: 'Created pod: splitter-graph-3bba6-879cd4d78-6rx4b' metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:15Z" name: splitter-graph-3bba6-879cd4d78.18d5e34f01fa806a namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 11692942-12eb-479d-8d81-ec7257da72e8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-3bba6 namespace: kserve-ci-e2e-test resourceVersion: "26451" uid: 83e94506-a74f-447e-9767-d81f1c6522f7 kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: Scaled up replica set splitter-graph-3bba6-879cd4d78 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:15Z" name: splitter-graph-3bba6.18d5e34f013119f9 namespace: kserve-ci-e2e-test resourceVersion: "26454" uid: d8f4800f-4288-4a0d-9a87-863d99fd5219 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-3bba6 namespace: kserve-ci-e2e-test resourceVersion: "26472" uid: be687d9f-2975-4a42-b7bc-19d287fe0a3e kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: InferenceGraph [splitter-graph-3bba6] is Ready metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:19:15Z" name: splitter-graph-3bba6.18d5e34f07389adb namespace: kserve-ci-e2e-test resourceVersion: "26474" uid: 31acadfa-5ece-4946-89d6-ea7e12e3ab19 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:19:25.712998Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq namespace: kserve-ci-e2e-test resourceVersion: "26681" uid: 730ef7f6-6f92-412d-8346-bcd0ca6c3264 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-17a4d-predictor-6749977b8c-bxntq to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:19:25Z" name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq.18d5e3516734c4c4 namespace: kserve-ci-e2e-test resourceVersion: "26687" uid: aa3e42a0-9181-4257-81c1-bd8bdd5dfb71 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 730ef7f6-6f92-412d-8346-bcd0ca6c3264 kind: Event lastTimestamp: "2026-09-16T19:19:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-17a4d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:25Z" name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq.18d5e351766b38aa namespace: kserve-ci-e2e-test resourceVersion: "26722" uid: 6e2ff1a8-6e80-4c7e-8e0f-a4a061b9ab5f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: 730ef7f6-6f92-412d-8346-bcd0ca6c3264 kind: Event lastTimestamp: "2026-09-16T19:19:26Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:19:26Z" name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq.18d5e351acc6d81d namespace: kserve-ci-e2e-test resourceVersion: "26735" uid: 59e06725-64bd-4d84-811b-e50640397240 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 730ef7f6-6f92-412d-8346-bcd0ca6c3264 kind: Event lastTimestamp: "2026-09-16T19:19:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:26Z" name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq.18d5e351ae13f95e namespace: kserve-ci-e2e-test resourceVersion: "26737" uid: 793e1d10-2678-4a81-89ee-2b18d28681e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 730ef7f6-6f92-412d-8346-bcd0ca6c3264 kind: Event lastTimestamp: "2026-09-16T19:19:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:26Z" name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq.18d5e351b0e653c8 namespace: kserve-ci-e2e-test resourceVersion: "26738" uid: cef5cb88-2d0c-4b52-ab97-efa28bc026e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 730ef7f6-6f92-412d-8346-bcd0ca6c3264 kind: Event lastTimestamp: "2026-09-16T19:19:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:26Z" name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq.18d5e351b15340b8 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: e08a5f99-f366-4d1a-9d29-204c3650fc18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 730ef7f6-6f92-412d-8346-bcd0ca6c3264 kind: Event lastTimestamp: "2026-09-16T19:19:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:26Z" name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq.18d5e351b15e5565 namespace: kserve-ci-e2e-test resourceVersion: "26740" uid: 3851eaaf-4cae-48e4-aa06-228ff99a5bdc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 730ef7f6-6f92-412d-8346-bcd0ca6c3264 kind: Event lastTimestamp: "2026-09-16T19:19:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:27Z" name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq.18d5e351b44cc475 namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: 8193a3e7-574f-45b6-8200-3cc62044be11 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 730ef7f6-6f92-412d-8346-bcd0ca6c3264 kind: Event lastTimestamp: "2026-09-16T19:19:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:27Z" name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq.18d5e351b4b7e279 namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: 617488c3-fc5f-4b1a-aab4-390f5123d3c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:19:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 730ef7f6-6f92-412d-8346-bcd0ca6c3264 kind: Event lastTimestamp: "2026-09-16T19:20:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:19:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:20:03Z" name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq.18d5e351bfcce62b namespace: kserve-ci-e2e-test resourceVersion: "27073" uid: 2c674944-7c14-4781-8f1e-9a2c843a1b82 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 730ef7f6-6f92-412d-8346-bcd0ca6c3264 kind: Event lastTimestamp: "2026-09-16T19:28:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:28:40Z" name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq.18d5e3d28b85264b namespace: kserve-ci-e2e-test resourceVersion: "30795" uid: d04677a5-337f-4348-8c6f-1f80652936c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: 730ef7f6-6f92-412d-8346-bcd0ca6c3264 kind: Event lastTimestamp: "2026-09-16T19:28:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:28:40Z" name: success-200-isvc-17a4d-predictor-6749977b8c-bxntq.18d5e3d28b8589af namespace: kserve-ci-e2e-test resourceVersion: "30800" uid: 1046bd11-0fda-424c-b907-23bcd054885e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-17a4d-predictor-6749977b8c namespace: kserve-ci-e2e-test resourceVersion: "26676" uid: 7df61efa-d701-4e07-9a7e-0581e1c64f17 kind: Event lastTimestamp: "2026-09-16T19:19:25Z" message: 'Created pod: success-200-isvc-17a4d-predictor-6749977b8c-bxntq' metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:25Z" name: success-200-isvc-17a4d-predictor-6749977b8c.18d5e351672252ec namespace: kserve-ci-e2e-test resourceVersion: "26684" uid: c0970e55-bcab-4160-a73c-ae1256b0449d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-17a4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26674" uid: bf7b494c-0459-409e-b338-f5de082eb457 kind: Event lastTimestamp: "2026-09-16T19:19:25Z" message: Scaled up replica set success-200-isvc-17a4d-predictor-6749977b8c from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:25Z" name: success-200-isvc-17a4d-predictor.18d5e35165389f78 namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 6543b8a7-86a0-4bdb-b84f-67d5aace442f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:19:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-17a4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26675" uid: c1eba0ed-c007-40d2-bc52-0b3edbfd873f kind: Event lastTimestamp: "2026-09-16T19:19:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:19:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:55Z" name: success-200-isvc-17a4d-predictor.18d5e354e3e8f58c namespace: kserve-ci-e2e-test resourceVersion: "27009" uid: 956b0e3e-9aa9-46f5-aaa2-385f80a5b252 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:19:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-17a4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26675" uid: c1eba0ed-c007-40d2-bc52-0b3edbfd873f kind: Event lastTimestamp: "2026-09-16T19:19:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:19:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:55Z" name: success-200-isvc-17a4d-predictor.18d5e354e4686c83 namespace: kserve-ci-e2e-test resourceVersion: "27010" uid: 61a486a3-0bac-4261-9717-134132d4a232 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:20:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-17a4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26877" uid: c1eba0ed-c007-40d2-bc52-0b3edbfd873f kind: Event lastTimestamp: "2026-09-16T19:20:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:20:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:40Z" name: success-200-isvc-17a4d-predictor.18d5e35be1f84048 namespace: kserve-ci-e2e-test resourceVersion: "27468" uid: ae5c6585-6aef-49e8-9765-367299ba3b8f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:20:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-17a4d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26877" uid: c1eba0ed-c007-40d2-bc52-0b3edbfd873f kind: Event lastTimestamp: "2026-09-16T19:20:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:20:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:40Z" name: success-200-isvc-17a4d-predictor.18d5e35be24ec30e namespace: kserve-ci-e2e-test resourceVersion: "27469" uid: f99bf6f6-c357-4f36-b041-7dff23dea12e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:19:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-17a4d namespace: kserve-ci-e2e-test resourceVersion: "26662" uid: 35f2f114-1f93-4df3-8700-5f36e7bdbfc7 kind: Event lastTimestamp: "2026-09-16T19:20:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:20:13Z" name: success-200-isvc-17a4d.18d5e351653c6ff4 namespace: kserve-ci-e2e-test resourceVersion: "27175" uid: b160b7f0-a180-4099-843e-e93ed0b76c40 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-17a4d namespace: kserve-ci-e2e-test resourceVersion: "27172" uid: 35f2f114-1f93-4df3-8700-5f36e7bdbfc7 kind: Event lastTimestamp: "2026-09-16T19:20:13Z" message: InferenceService [success-200-isvc-17a4d] is Ready metadata: creationTimestamp: "2026-09-16T19:20:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:20:13Z" name: success-200-isvc-17a4d.18d5e35c7da2fb19 namespace: kserve-ci-e2e-test resourceVersion: "27173" uid: fd02005f-59fa-47dc-bc4a-763147a76b74 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:09:00.582914Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-27911-predictor-8554d7997-sc9n6 namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: 4f64ec37-211a-4ebe-9a29-f20a249332d0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-27911-predictor-8554d7997-sc9n6 to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:09:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:09:00Z" name: success-200-isvc-27911-predictor-8554d7997-sc9n6.18d5e2bfda8cad6b namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: 7a589aae-a237-4e1c-8e6b-1399abb1dc66 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-27911-predictor-8554d7997-sc9n6 namespace: kserve-ci-e2e-test resourceVersion: "21146" uid: 4f64ec37-211a-4ebe-9a29-f20a249332d0 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:09:01Z" name: success-200-isvc-27911-predictor-8554d7997-sc9n6.18d5e2bffc2a52d5 namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: a45a9dbc-fe1b-42ca-9c08-b3dcac03f63d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-27911-predictor-8554d7997-sc9n6 namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: 4f64ec37-211a-4ebe-9a29-f20a249332d0 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:01Z" name: success-200-isvc-27911-predictor-8554d7997-sc9n6.18d5e2bffdc41a4d namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: 92295f84-d526-48f2-af56-5151d252a72f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-27911-predictor-8554d7997-sc9n6 namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: 4f64ec37-211a-4ebe-9a29-f20a249332d0 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:01Z" name: success-200-isvc-27911-predictor-8554d7997-sc9n6.18d5e2c0008d017e namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: 25d0b86b-924a-47e5-a748-e7a0bce6fc90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-27911-predictor-8554d7997-sc9n6 namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: 4f64ec37-211a-4ebe-9a29-f20a249332d0 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:01Z" name: success-200-isvc-27911-predictor-8554d7997-sc9n6.18d5e2c000f8f66f namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: 70101eeb-3751-451d-bdbc-73ac186939b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-27911-predictor-8554d7997-sc9n6 namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: 4f64ec37-211a-4ebe-9a29-f20a249332d0 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:01Z" name: success-200-isvc-27911-predictor-8554d7997-sc9n6.18d5e2c001030362 namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: e9b99020-d48c-4806-a1d0-61d724bf8bfd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-27911-predictor-8554d7997-sc9n6 namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: 4f64ec37-211a-4ebe-9a29-f20a249332d0 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:01Z" name: success-200-isvc-27911-predictor-8554d7997-sc9n6.18d5e2c003f49bac namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: fe049653-bead-4c89-bcc2-238aba22c411 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-27911-predictor-8554d7997-sc9n6 namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: 4f64ec37-211a-4ebe-9a29-f20a249332d0 kind: Event lastTimestamp: "2026-09-16T19:09:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:01Z" name: success-200-isvc-27911-predictor-8554d7997-sc9n6.18d5e2c004602c88 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 0beb610d-f752-4e02-ac45-297f19e77068 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T19:09:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-27911-predictor-8554d7997-sc9n6 namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: 4f64ec37-211a-4ebe-9a29-f20a249332d0 kind: Event lastTimestamp: "2026-09-16T19:18:17Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:09:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:17Z" name: success-200-isvc-27911-predictor-8554d7997-sc9n6.18d5e2c026f0223c namespace: kserve-ci-e2e-test resourceVersion: "25875" uid: 4fdb7b38-a986-489e-9a38-c814f93a69d3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-27911-predictor-8554d7997-sc9n6 namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: 4f64ec37-211a-4ebe-9a29-f20a249332d0 kind: Event lastTimestamp: "2026-09-16T19:18:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:15Z" name: success-200-isvc-27911-predictor-8554d7997-sc9n6.18d5e34105530090 namespace: kserve-ci-e2e-test resourceVersion: "25763" uid: d8c33f21-6422-49ff-a2f9-93bc68e73e06 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-27911-predictor-8554d7997-sc9n6 namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: 4f64ec37-211a-4ebe-9a29-f20a249332d0 kind: Event lastTimestamp: "2026-09-16T19:18:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:15Z" name: success-200-isvc-27911-predictor-8554d7997-sc9n6.18d5e3410553734d namespace: kserve-ci-e2e-test resourceVersion: "25768" uid: 6e7882f7-7ba1-4aeb-bec6-80b154e55a90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-27911-predictor-8554d7997-sc9n6 namespace: kserve-ci-e2e-test resourceVersion: "21141" uid: 4f64ec37-211a-4ebe-9a29-f20a249332d0 kind: Event lastTimestamp: "2026-09-16T19:18:17Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:17Z" name: success-200-isvc-27911-predictor-8554d7997-sc9n6.18d5e3419b56f70b namespace: kserve-ci-e2e-test resourceVersion: "25874" uid: 494afb2f-bcb1-4f57-9979-6aa92c593891 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-27911-predictor-8554d7997 namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: aa56b330-9bc3-41f6-a91d-42c85d754d6d kind: Event lastTimestamp: "2026-09-16T19:09:00Z" message: 'Created pod: success-200-isvc-27911-predictor-8554d7997-sc9n6' metadata: creationTimestamp: "2026-09-16T19:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:00Z" name: success-200-isvc-27911-predictor-8554d7997.18d5e2bfd90dc003 namespace: kserve-ci-e2e-test resourceVersion: "21139" uid: 17e9e41a-28f7-4021-abb5-8be0703d311d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-27911-predictor namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: 28ff4f97-6f15-4df1-8a55-01e38602a7ce kind: Event lastTimestamp: "2026-09-16T19:09:00Z" message: Scaled up replica set success-200-isvc-27911-predictor-8554d7997 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:00Z" name: success-200-isvc-27911-predictor.18d5e2bfd673a696 namespace: kserve-ci-e2e-test resourceVersion: "21131" uid: 71853950-9384-4b3a-9851-45b6118db106 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:09:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-27911-predictor namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 5679e28b-0a11-4d6c-be85-586b2085d9df kind: Event lastTimestamp: "2026-09-16T19:09:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:09:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:30Z" name: success-200-isvc-27911-predictor.18d5e2c354e64e4b namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 5f593ee3-b9f1-4449-a246-f5c151838d1c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:09:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-27911-predictor namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 5679e28b-0a11-4d6c-be85-586b2085d9df kind: Event lastTimestamp: "2026-09-16T19:09:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:09:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:30Z" name: success-200-isvc-27911-predictor.18d5e2c35538d866 namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 8133553a-ab3b-4df8-b533-e140d7bd7937 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:09:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-27911-predictor namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: 5679e28b-0a11-4d6c-be85-586b2085d9df kind: Event lastTimestamp: "2026-09-16T19:10:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:09:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:15Z" name: success-200-isvc-27911-predictor.18d5e2ca53dabb80 namespace: kserve-ci-e2e-test resourceVersion: "22082" uid: 9eba8ba9-e224-4548-9860-ac77f0d437ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:09:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-27911-predictor namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: 5679e28b-0a11-4d6c-be85-586b2085d9df kind: Event lastTimestamp: "2026-09-16T19:10:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:09:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:15Z" name: success-200-isvc-27911-predictor.18d5e2ca5461de2b namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: 9d341d83-8de2-46e2-925c-1d58e983581a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:09:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-27911 namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 68f2f928-f87b-4172-a505-b122e36e831b kind: Event lastTimestamp: "2026-09-16T19:09:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:09:47Z" name: success-200-isvc-27911.18d5e2bfd69b1786 namespace: kserve-ci-e2e-test resourceVersion: "21626" uid: 75a4af06-3e9f-4e75-a8f3-8a7e384e240d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-27911 namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: 68f2f928-f87b-4172-a505-b122e36e831b kind: Event lastTimestamp: "2026-09-16T19:09:47Z" message: InferenceService [success-200-isvc-27911] is Ready metadata: creationTimestamp: "2026-09-16T19:09:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:09:47Z" name: success-200-isvc-27911.18d5e2cae26bae9a namespace: kserve-ci-e2e-test resourceVersion: "21623" uid: 05e028bf-d50d-4f0e-91da-6f18d9b86c90 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:18:15.468112Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj namespace: kserve-ci-e2e-test resourceVersion: "25780" uid: 6d7380ca-84ba-43db-be17-cc87934737ac kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3bba6-predictor-65498c7465-jfhbj to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:18:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:18:15Z" name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj.18d5e3410c48e32d namespace: kserve-ci-e2e-test resourceVersion: "25790" uid: abaa49da-62ce-4208-83d4-1c7a4f022e99 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj namespace: kserve-ci-e2e-test resourceVersion: "25803" uid: 6d7380ca-84ba-43db-be17-cc87934737ac kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:18:16Z" name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj.18d5e3412e408ce6 namespace: kserve-ci-e2e-test resourceVersion: "25841" uid: 8b4461c3-134f-445d-9302-7e47ada37195 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: 6d7380ca-84ba-43db-be17-cc87934737ac kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:16Z" name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj.18d5e3412fb62ad4 namespace: kserve-ci-e2e-test resourceVersion: "25843" uid: 89c2f14d-7f68-4362-85fe-1d1da700467d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: 6d7380ca-84ba-43db-be17-cc87934737ac kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:16Z" name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj.18d5e34132917cc7 namespace: kserve-ci-e2e-test resourceVersion: "25844" uid: a81956b8-302b-4943-9450-3ea41fe2c16c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: 6d7380ca-84ba-43db-be17-cc87934737ac kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:16Z" name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj.18d5e3413301d0e7 namespace: kserve-ci-e2e-test resourceVersion: "25845" uid: d5b7b3f3-8d5a-43e7-8026-2d73240f9ff9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: 6d7380ca-84ba-43db-be17-cc87934737ac kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:16Z" name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj.18d5e341330b0cf4 namespace: kserve-ci-e2e-test resourceVersion: "25846" uid: 79dbb254-9024-4307-9532-5e06ff4adfe6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: 6d7380ca-84ba-43db-be17-cc87934737ac kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:16Z" name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj.18d5e34135669b7d namespace: kserve-ci-e2e-test resourceVersion: "25847" uid: 2b17220b-d27a-43bd-89ec-c70cf4e40de2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: 6d7380ca-84ba-43db-be17-cc87934737ac kind: Event lastTimestamp: "2026-09-16T19:18:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:18:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:16Z" name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj.18d5e34135ebd586 namespace: kserve-ci-e2e-test resourceVersion: "25848" uid: b47b645a-eb53-45e7-b46b-59fc27a19871 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:18:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: 6d7380ca-84ba-43db-be17-cc87934737ac kind: Event lastTimestamp: "2026-09-16T19:18:53Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:18:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:18:53Z" name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj.18d5e3419f00b1fa namespace: kserve-ci-e2e-test resourceVersion: "26170" uid: 1eaf5f0c-997b-4521-8bf8-5398be790ae9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: 6d7380ca-84ba-43db-be17-cc87934737ac kind: Event lastTimestamp: "2026-09-16T19:19:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:25Z" name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj.18d5e3515fe4048e namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: 3c9e41ae-36c4-4180-9a3b-8b856b3b0033 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: 6d7380ca-84ba-43db-be17-cc87934737ac kind: Event lastTimestamp: "2026-09-16T19:19:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:25Z" name: success-200-isvc-3bba6-predictor-65498c7465-jfhbj.18d5e3515fe48616 namespace: kserve-ci-e2e-test resourceVersion: "26649" uid: fa5d65b4-8821-4b28-b9b9-e278e9dc4970 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3bba6-predictor-65498c7465 namespace: kserve-ci-e2e-test resourceVersion: "25773" uid: 75521b8a-84cc-4cfe-bd8b-3c49af13e9b2 kind: Event lastTimestamp: "2026-09-16T19:18:15Z" message: 'Created pod: success-200-isvc-3bba6-predictor-65498c7465-jfhbj' metadata: creationTimestamp: "2026-09-16T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:15Z" name: success-200-isvc-3bba6-predictor-65498c7465.18d5e3410bce8dc0 namespace: kserve-ci-e2e-test resourceVersion: "25786" uid: f17cd47c-590b-4791-b848-0abc70305703 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:18:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3bba6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25769" uid: eb44f024-8a4f-48ee-b90e-14ebc37496ef kind: Event lastTimestamp: "2026-09-16T19:18:15Z" message: Scaled up replica set success-200-isvc-3bba6-predictor-65498c7465 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:15Z" name: success-200-isvc-3bba6-predictor.18d5e34108626a86 namespace: kserve-ci-e2e-test resourceVersion: "25774" uid: 014df9d7-c8ab-4871-985c-a83a5ac0013a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:18:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3bba6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: d2e83cdb-1734-4f2e-8841-1cdf3b3444ff kind: Event lastTimestamp: "2026-09-16T19:18:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:18:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:45Z" name: success-200-isvc-3bba6-predictor.18d5e34488199723 namespace: kserve-ci-e2e-test resourceVersion: "26103" uid: c33fdd22-3e6c-4376-bcaf-8117e8c87410 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:18:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3bba6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25772" uid: d2e83cdb-1734-4f2e-8841-1cdf3b3444ff kind: Event lastTimestamp: "2026-09-16T19:18:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:18:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:18:45Z" name: success-200-isvc-3bba6-predictor.18d5e344888d9834 namespace: kserve-ci-e2e-test resourceVersion: "26104" uid: d06f1e15-01da-4da7-a2fd-215bc5ef1791 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:19:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3bba6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25985" uid: d2e83cdb-1734-4f2e-8841-1cdf3b3444ff kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:19:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:15Z" name: success-200-isvc-3bba6-predictor.18d5e34b86f1253b namespace: kserve-ci-e2e-test resourceVersion: "26468" uid: 3085181e-addd-406f-97d6-029e180431af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:19:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3bba6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25985" uid: d2e83cdb-1734-4f2e-8841-1cdf3b3444ff kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:19:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:15Z" name: success-200-isvc-3bba6-predictor.18d5e34b876f520d namespace: kserve-ci-e2e-test resourceVersion: "26471" uid: 9c16b77a-4a1a-4203-b509-fd382e2e27eb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T19:18:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3bba6 namespace: kserve-ci-e2e-test resourceVersion: "25760" uid: d9f29519-5496-4ddd-9dcc-565db15842fb kind: Event lastTimestamp: "2026-09-16T19:19:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:19:04Z" name: success-200-isvc-3bba6.18d5e34108a754e3 namespace: kserve-ci-e2e-test resourceVersion: "26261" uid: 2d7bac47-dc53-4f3f-9710-2718dc885ef0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3bba6 namespace: kserve-ci-e2e-test resourceVersion: "26259" uid: d9f29519-5496-4ddd-9dcc-565db15842fb kind: Event lastTimestamp: "2026-09-16T19:19:04Z" message: InferenceService [success-200-isvc-3bba6] is Ready metadata: creationTimestamp: "2026-09-16T19:19:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:19:04Z" name: success-200-isvc-3bba6.18d5e34c5b11f06c namespace: kserve-ci-e2e-test resourceVersion: "26260" uid: 74e50316-18f1-4e00-b6f0-e35430b42ca4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:07:50.231814Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-443e6-predictor-c76bc877c-x6z66 namespace: kserve-ci-e2e-test resourceVersion: "20265" uid: 8100d59c-40c8-4523-8dd5-e31a6c5cb54b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-443e6-predictor-c76bc877c-x6z66 to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:07:50Z" name: success-200-isvc-443e6-predictor-c76bc877c-x6z66.18d5e2af794c170b namespace: kserve-ci-e2e-test resourceVersion: "20274" uid: 845fbca5-d0ef-4a9b-89e3-642e21e9c32b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-443e6-predictor-c76bc877c-x6z66 namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8100d59c-40c8-4523-8dd5-e31a6c5cb54b kind: Event lastTimestamp: "2026-09-16T19:07:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-443e6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:50Z" name: success-200-isvc-443e6-predictor-c76bc877c-x6z66.18d5e2af82e946b3 namespace: kserve-ci-e2e-test resourceVersion: "20304" uid: 37db2422-bc6d-409d-bf76-670489071020 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-443e6-predictor-c76bc877c-x6z66 namespace: kserve-ci-e2e-test resourceVersion: "20281" uid: 8100d59c-40c8-4523-8dd5-e31a6c5cb54b kind: Event lastTimestamp: "2026-09-16T19:07:51Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:07:51Z" name: success-200-isvc-443e6-predictor-c76bc877c-x6z66.18d5e2afbee36a14 namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 6331f700-a867-4ba7-b837-4d9b7bd006c3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-443e6-predictor-c76bc877c-x6z66 namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8100d59c-40c8-4523-8dd5-e31a6c5cb54b kind: Event lastTimestamp: "2026-09-16T19:07:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:51Z" name: success-200-isvc-443e6-predictor-c76bc877c-x6z66.18d5e2afc03c1f06 namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: f6f1eb11-463f-4338-b77f-02cdc3bfa468 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-443e6-predictor-c76bc877c-x6z66 namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8100d59c-40c8-4523-8dd5-e31a6c5cb54b kind: Event lastTimestamp: "2026-09-16T19:07:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:51Z" name: success-200-isvc-443e6-predictor-c76bc877c-x6z66.18d5e2afc32c0007 namespace: kserve-ci-e2e-test resourceVersion: "20333" uid: 54989222-5603-4b14-99b2-5cc548cdcf97 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-443e6-predictor-c76bc877c-x6z66 namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8100d59c-40c8-4523-8dd5-e31a6c5cb54b kind: Event lastTimestamp: "2026-09-16T19:07:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:51Z" name: success-200-isvc-443e6-predictor-c76bc877c-x6z66.18d5e2afc3ab5b3d namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: c61a6030-1751-42f0-baf4-508e2e49c8e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-443e6-predictor-c76bc877c-x6z66 namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8100d59c-40c8-4523-8dd5-e31a6c5cb54b kind: Event lastTimestamp: "2026-09-16T19:07:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:51Z" name: success-200-isvc-443e6-predictor-c76bc877c-x6z66.18d5e2afc3b4a695 namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: 83fe44a8-f4f0-4185-8d75-20d87f343610 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-443e6-predictor-c76bc877c-x6z66 namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8100d59c-40c8-4523-8dd5-e31a6c5cb54b kind: Event lastTimestamp: "2026-09-16T19:07:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:51Z" name: success-200-isvc-443e6-predictor-c76bc877c-x6z66.18d5e2afc676325b namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: c554200d-787b-4e8d-be04-182f6f0c48e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-443e6-predictor-c76bc877c-x6z66 namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8100d59c-40c8-4523-8dd5-e31a6c5cb54b kind: Event lastTimestamp: "2026-09-16T19:07:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:51Z" name: success-200-isvc-443e6-predictor-c76bc877c-x6z66.18d5e2afc6e3e06d namespace: kserve-ci-e2e-test resourceVersion: "20337" uid: 987c10d9-3005-4a95-a403-c21080346358 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:07:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-443e6-predictor-c76bc877c-x6z66 namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8100d59c-40c8-4523-8dd5-e31a6c5cb54b kind: Event lastTimestamp: "2026-09-16T19:08:28Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:28Z" name: success-200-isvc-443e6-predictor-c76bc877c-x6z66.18d5e2b00672316f namespace: kserve-ci-e2e-test resourceVersion: "20630" uid: 9b595de5-7484-446d-b13c-81756d31c154 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-443e6-predictor-c76bc877c-x6z66 namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8100d59c-40c8-4523-8dd5-e31a6c5cb54b kind: Event lastTimestamp: "2026-09-16T19:09:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:00Z" name: success-200-isvc-443e6-predictor-c76bc877c-x6z66.18d5e2bfd247abdb namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: 944b7adb-a855-4554-90e5-926ea3df6b74 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-443e6-predictor-c76bc877c-x6z66 namespace: kserve-ci-e2e-test resourceVersion: "20268" uid: 8100d59c-40c8-4523-8dd5-e31a6c5cb54b kind: Event lastTimestamp: "2026-09-16T19:09:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:09:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:00Z" name: success-200-isvc-443e6-predictor-c76bc877c-x6z66.18d5e2bfd248c350 namespace: kserve-ci-e2e-test resourceVersion: "21121" uid: 237a4c2a-c636-4a35-8253-7045ec43a0d2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-443e6-predictor-c76bc877c namespace: kserve-ci-e2e-test resourceVersion: "20240" uid: 1db34e46-f72c-45da-b44b-8c2214d54de6 kind: Event lastTimestamp: "2026-09-16T19:07:50Z" message: 'Created pod: success-200-isvc-443e6-predictor-c76bc877c-x6z66' metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:50Z" name: success-200-isvc-443e6-predictor-c76bc877c.18d5e2af78ca8021 namespace: kserve-ci-e2e-test resourceVersion: "20266" uid: b4f6725e-8096-44dc-a99f-48bbeb110d6f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-443e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20234" uid: be3631b5-7abd-4de4-a8ee-7e38f1e2c5e6 kind: Event lastTimestamp: "2026-09-16T19:07:50Z" message: Scaled up replica set success-200-isvc-443e6-predictor-c76bc877c from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:07:50Z" name: success-200-isvc-443e6-predictor.18d5e2af747eec56 namespace: kserve-ci-e2e-test resourceVersion: "20243" uid: dffa36ed-8df1-41d7-aef9-ccca71c5b6c4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:08:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-443e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20237" uid: c1feccb5-5358-41bd-905b-2bd0d35a32ac kind: Event lastTimestamp: "2026-09-16T19:08:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:35Z" name: success-200-isvc-443e6-predictor.18d5e2b2f35b7cc0 namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: bdc02b8c-b2d3-44d3-a937-49f96f8931d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:08:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-443e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20237" uid: c1feccb5-5358-41bd-905b-2bd0d35a32ac kind: Event lastTimestamp: "2026-09-16T19:08:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:35Z" name: success-200-isvc-443e6-predictor.18d5e2b2f3bce00f namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: db251f69-b263-4f25-9352-901ffbf4b15d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-443e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: c1feccb5-5358-41bd-905b-2bd0d35a32ac kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:50Z" name: success-200-isvc-443e6-predictor.18d5e2bd710e74e5 namespace: kserve-ci-e2e-test resourceVersion: "20927" uid: d2df513e-c978-4dce-8b55-c4ac14c92123 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-443e6-predictor namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: c1feccb5-5358-41bd-905b-2bd0d35a32ac kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:50Z" name: success-200-isvc-443e6-predictor.18d5e2bd717ea991 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: 1e30e242-2679-48cc-82a7-b9083389f8fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:07:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-443e6 namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: 661299ee-406d-417a-bfc9-c7b8092cf461 kind: Event lastTimestamp: "2026-09-16T19:08:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:08:38Z" name: success-200-isvc-443e6.18d5e2af74757f89 namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: 085e77f4-60ac-4452-8df9-8fd200d59fb2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-443e6 namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: 661299ee-406d-417a-bfc9-c7b8092cf461 kind: Event lastTimestamp: "2026-09-16T19:08:38Z" message: InferenceService [success-200-isvc-443e6] is Ready metadata: creationTimestamp: "2026-09-16T19:08:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:08:38Z" name: success-200-isvc-443e6.18d5e2bac23f2e8a namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: a13da0d7-5d38-40b7-9558-63fabb839fcf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:10:00.254709Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-53a10-predictor-86b75c99c-66zjg namespace: kserve-ci-e2e-test resourceVersion: "21832" uid: 9b66c1c9-85af-4ec4-a37b-9ac29daca655 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-53a10-predictor-86b75c99c-66zjg to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:10:00Z" name: success-200-isvc-53a10-predictor-86b75c99c-66zjg.18d5e2cdbf440601 namespace: kserve-ci-e2e-test resourceVersion: "21836" uid: c96abda3-ff20-4fda-ac4c-228aa311a5e1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-53a10-predictor-86b75c99c-66zjg namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 9b66c1c9-85af-4ec4-a37b-9ac29daca655 kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-53a10-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:00Z" name: success-200-isvc-53a10-predictor-86b75c99c-66zjg.18d5e2cdc7e59779 namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: a6bb0fe2-fca3-4904-8eaf-c7fa913c8326 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-53a10-predictor-86b75c99c-66zjg namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 9b66c1c9-85af-4ec4-a37b-9ac29daca655 kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:10:01Z" name: success-200-isvc-53a10-predictor-86b75c99c-66zjg.18d5e2ce04d9079a namespace: kserve-ci-e2e-test resourceVersion: "21926" uid: 787fc52b-923a-4c39-8f54-32054eaf3499 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53a10-predictor-86b75c99c-66zjg namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 9b66c1c9-85af-4ec4-a37b-9ac29daca655 kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: success-200-isvc-53a10-predictor-86b75c99c-66zjg.18d5e2ce0634345b namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: adffc83b-d51e-45b4-848d-01b26af62a12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53a10-predictor-86b75c99c-66zjg namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 9b66c1c9-85af-4ec4-a37b-9ac29daca655 kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: success-200-isvc-53a10-predictor-86b75c99c-66zjg.18d5e2ce08c01d08 namespace: kserve-ci-e2e-test resourceVersion: "21929" uid: f32d3090-71ed-4e40-a46f-878cf6d654d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53a10-predictor-86b75c99c-66zjg namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 9b66c1c9-85af-4ec4-a37b-9ac29daca655 kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: success-200-isvc-53a10-predictor-86b75c99c-66zjg.18d5e2ce0931d163 namespace: kserve-ci-e2e-test resourceVersion: "21930" uid: e2ce20ec-52dd-4fba-8102-e1c6dceaa581 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53a10-predictor-86b75c99c-66zjg namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 9b66c1c9-85af-4ec4-a37b-9ac29daca655 kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: success-200-isvc-53a10-predictor-86b75c99c-66zjg.18d5e2ce093bbc9e namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: 215670f1-09b5-4067-829d-f522da585eef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53a10-predictor-86b75c99c-66zjg namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 9b66c1c9-85af-4ec4-a37b-9ac29daca655 kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: success-200-isvc-53a10-predictor-86b75c99c-66zjg.18d5e2ce0bb64598 namespace: kserve-ci-e2e-test resourceVersion: "21932" uid: 5f0b5d75-a903-40a8-a080-f0d709b11068 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53a10-predictor-86b75c99c-66zjg namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 9b66c1c9-85af-4ec4-a37b-9ac29daca655 kind: Event lastTimestamp: "2026-09-16T19:10:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:01Z" name: success-200-isvc-53a10-predictor-86b75c99c-66zjg.18d5e2ce0c27b83b namespace: kserve-ci-e2e-test resourceVersion: "21933" uid: e2fc0dcd-d3e3-47cf-9f91-c293821884be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:10:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53a10-predictor-86b75c99c-66zjg namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 9b66c1c9-85af-4ec4-a37b-9ac29daca655 kind: Event lastTimestamp: "2026-09-16T19:10:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:10:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:38Z" name: success-200-isvc-53a10-predictor-86b75c99c-66zjg.18d5e2ce2d229208 namespace: kserve-ci-e2e-test resourceVersion: "22252" uid: 00fe1daa-ceda-4b6f-9644-1cc055e51259 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-53a10-predictor-86b75c99c-66zjg namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 9b66c1c9-85af-4ec4-a37b-9ac29daca655 kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:15Z" name: success-200-isvc-53a10-predictor-86b75c99c-66zjg.18d5e34eef06b9c4 namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: 886b83a4-7c6c-4dd9-9216-81b37752bfc6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-53a10-predictor-86b75c99c-66zjg namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 9b66c1c9-85af-4ec4-a37b-9ac29daca655 kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:15Z" name: success-200-isvc-53a10-predictor-86b75c99c-66zjg.18d5e34eef075878 namespace: kserve-ci-e2e-test resourceVersion: "26382" uid: 90b7eac3-5d3e-4ca5-950f-5248854eec75 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-53a10-predictor-86b75c99c namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: 48896c1a-d6d3-4669-86c6-79035f9c47f8 kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: 'Created pod: success-200-isvc-53a10-predictor-86b75c99c-66zjg' metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:00Z" name: success-200-isvc-53a10-predictor-86b75c99c.18d5e2cdbecccb84 namespace: kserve-ci-e2e-test resourceVersion: "21833" uid: 0fa750f1-8d93-4f7c-8fc7-029397c606d0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-53a10-predictor namespace: kserve-ci-e2e-test resourceVersion: "21821" uid: dbedfed2-2d6e-441f-bd52-45bddbae61dc kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: Scaled up replica set success-200-isvc-53a10-predictor-86b75c99c from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:00Z" name: success-200-isvc-53a10-predictor.18d5e2cdbcfb79fa namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: 0e80ff0f-1ba1-4ddc-a027-6646a179b172 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:10:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-53a10-predictor namespace: kserve-ci-e2e-test resourceVersion: "21823" uid: 51b5e20f-df66-45ba-9fc5-d0b723e00a6b kind: Event lastTimestamp: "2026-09-16T19:10:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:30Z" name: success-200-isvc-53a10-predictor.18d5e2d13b912741 namespace: kserve-ci-e2e-test resourceVersion: "22186" uid: a69708c5-c389-47dd-af7a-1388c654f820 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:10:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-53a10-predictor namespace: kserve-ci-e2e-test resourceVersion: "21823" uid: 51b5e20f-df66-45ba-9fc5-d0b723e00a6b kind: Event lastTimestamp: "2026-09-16T19:10:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:10:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:10:30Z" name: success-200-isvc-53a10-predictor.18d5e2d13c0ab0cd namespace: kserve-ci-e2e-test resourceVersion: "22187" uid: 35119e86-9130-4914-ab50-62aa1e6b2a11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:10:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-53a10-predictor namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: 51b5e20f-df66-45ba-9fc5-d0b723e00a6b kind: Event lastTimestamp: "2026-09-16T19:11:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:10:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:11:30Z" name: success-200-isvc-53a10-predictor.18d5e2d83a450688 namespace: kserve-ci-e2e-test resourceVersion: "22685" uid: 5606c54e-c6ee-433f-92fb-5d5920285d62 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-16T19:10:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-53a10-predictor namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: 51b5e20f-df66-45ba-9fc5-d0b723e00a6b kind: Event lastTimestamp: "2026-09-16T19:11:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:10:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:11:30Z" name: success-200-isvc-53a10-predictor.18d5e2d83aaf61d4 namespace: kserve-ci-e2e-test resourceVersion: "22686" uid: ddc836b4-e8cd-4fa7-a0a8-771c9abe2e1d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-53a10 namespace: kserve-ci-e2e-test resourceVersion: "21810" uid: 8767106d-dd91-4c20-aa91-25587413edbb kind: Event lastTimestamp: "2026-09-16T19:10:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:10:48Z" name: success-200-isvc-53a10.18d5e2cdbcf023ba namespace: kserve-ci-e2e-test resourceVersion: "22346" uid: 5042b9ff-65ca-461b-879d-cd43aceaa155 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-53a10 namespace: kserve-ci-e2e-test resourceVersion: "21810" uid: 8767106d-dd91-4c20-aa91-25587413edbb kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: 'Failed to update status for InferenceService "success-200-isvc-53a10": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-53a10": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:10:00Z" name: success-200-isvc-53a10.18d5e2cdbd5fa31d namespace: kserve-ci-e2e-test resourceVersion: "21829" uid: 360829b7-630a-40fe-94e5-626a742d7895 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-53a10 namespace: kserve-ci-e2e-test resourceVersion: "21810" uid: 8767106d-dd91-4c20-aa91-25587413edbb kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-53a10": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:10:00Z" name: success-200-isvc-53a10.18d5e2cdbd5fe859 namespace: kserve-ci-e2e-test resourceVersion: "21830" uid: e30dd056-3b85-417d-a0dc-f1fd110fa095 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-53a10 namespace: kserve-ci-e2e-test resourceVersion: "22342" uid: 8767106d-dd91-4c20-aa91-25587413edbb kind: Event lastTimestamp: "2026-09-16T19:10:48Z" message: InferenceService [success-200-isvc-53a10] is Ready metadata: creationTimestamp: "2026-09-16T19:10:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:10:48Z" name: success-200-isvc-53a10.18d5e2d8eb2843f8 namespace: kserve-ci-e2e-test resourceVersion: "22343" uid: 3c7d0611-d4b0-4426-a54a-daac5bd9a3fa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:19:15.247084Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd namespace: kserve-ci-e2e-test resourceVersion: "26408" uid: 743bd70b-3137-4d57-8c52-9c019a6aa05d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:19:15Z" name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd.18d5e34ef763a421 namespace: kserve-ci-e2e-test resourceVersion: "26413" uid: aeff1a89-9a46-4051-8390-60f68a9a237a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 743bd70b-3137-4d57-8c52-9c019a6aa05d kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7f5a4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:15Z" name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd.18d5e34effeb8834 namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: 32d4585f-c756-4ffc-ba6f-5aadab0b81d8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd namespace: kserve-ci-e2e-test resourceVersion: "26417" uid: 743bd70b-3137-4d57-8c52-9c019a6aa05d kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:19:16Z" name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd.18d5e34f3cd57cb1 namespace: kserve-ci-e2e-test resourceVersion: "26496" uid: 9f846655-d54f-42e4-aad1-3c8176b2e2fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 743bd70b-3137-4d57-8c52-9c019a6aa05d kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd.18d5e34f3e161024 namespace: kserve-ci-e2e-test resourceVersion: "26498" uid: 446dc4e5-1bbb-430e-880d-74fceff323f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 743bd70b-3137-4d57-8c52-9c019a6aa05d kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd.18d5e34f417c1233 namespace: kserve-ci-e2e-test resourceVersion: "26499" uid: 36cd6ebc-c20d-4e30-a33a-f2ed408b5d40 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 743bd70b-3137-4d57-8c52-9c019a6aa05d kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd.18d5e34f41e71999 namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: 7b40c9c9-818e-4a62-a18d-0cadf9dbbbef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 743bd70b-3137-4d57-8c52-9c019a6aa05d kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd.18d5e34f41f0ba00 namespace: kserve-ci-e2e-test resourceVersion: "26501" uid: 7038fb9c-b071-4571-9d3d-eaeed37cdce2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 743bd70b-3137-4d57-8c52-9c019a6aa05d kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd.18d5e34f45112992 namespace: kserve-ci-e2e-test resourceVersion: "26504" uid: 1e9dacdf-0b68-48c7-ba0d-3b002ef6e397 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 743bd70b-3137-4d57-8c52-9c019a6aa05d kind: Event lastTimestamp: "2026-09-16T19:19:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:19:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:16Z" name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd.18d5e34f45801a73 namespace: kserve-ci-e2e-test resourceVersion: "26506" uid: 1ed0097a-acbe-4cd1-89fd-93fd39d634af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:19:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 743bd70b-3137-4d57-8c52-9c019a6aa05d kind: Event lastTimestamp: "2026-09-16T19:19:53Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:19:53Z" name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd.18d5e34f6933f93d namespace: kserve-ci-e2e-test resourceVersion: "26987" uid: c1492315-f0db-4f48-8ac1-ff94750c8fd9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:36:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 743bd70b-3137-4d57-8c52-9c019a6aa05d kind: Event lastTimestamp: "2026-09-16T19:36:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:36:34Z" name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd.18d5e440e8ba518a namespace: kserve-ci-e2e-test resourceVersion: "34063" uid: d09eacb1-b5a8-4185-a61e-33ee10a0abd8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:36:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: 743bd70b-3137-4d57-8c52-9c019a6aa05d kind: Event lastTimestamp: "2026-09-16T19:36:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:36:34Z" name: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd.18d5e440e8bada6a namespace: kserve-ci-e2e-test resourceVersion: "34064" uid: eb7db14c-a51c-4639-8c45-3fdb1dcc5d9b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7f5a4-predictor-85677f6c9f namespace: kserve-ci-e2e-test resourceVersion: "26402" uid: 4e78ee05-ed35-4672-86a7-5e6a17a8ab14 kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: 'Created pod: success-200-isvc-7f5a4-predictor-85677f6c9f-gq7wd' metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:15Z" name: success-200-isvc-7f5a4-predictor-85677f6c9f.18d5e34ef6f4d653 namespace: kserve-ci-e2e-test resourceVersion: "26409" uid: 6c204f58-b097-4a12-93f9-a6a840425bba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7f5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "26400" uid: c88c07e2-0474-4fe5-852e-9d95ac9e64eb kind: Event lastTimestamp: "2026-09-16T19:19:15Z" message: Scaled up replica set success-200-isvc-7f5a4-predictor-85677f6c9f from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:15Z" name: success-200-isvc-7f5a4-predictor.18d5e34ef50ec321 namespace: kserve-ci-e2e-test resourceVersion: "26403" uid: 5dffacf7-972a-4fa5-abf9-67e386823ca5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:19:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7f5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "26401" uid: 3ef6225f-e433-4888-b7ae-2b0b1072d058 kind: Event lastTimestamp: "2026-09-16T19:19:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:45Z" name: success-200-isvc-7f5a4-predictor.18d5e35275215de5 namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 3d123526-eba2-4281-b4f2-955c5a18df44 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T19:19:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7f5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "26401" uid: 3ef6225f-e433-4888-b7ae-2b0b1072d058 kind: Event lastTimestamp: "2026-09-16T19:19:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:19:45Z" name: success-200-isvc-7f5a4-predictor.18d5e35275bff9e0 namespace: kserve-ci-e2e-test resourceVersion: "26930" uid: e5882f9f-bbd1-4733-8325-064eb94e2ea7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:20:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7f5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "26790" uid: 3ef6225f-e433-4888-b7ae-2b0b1072d058 kind: Event lastTimestamp: "2026-09-16T19:20:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:20:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:30Z" name: success-200-isvc-7f5a4-predictor.18d5e35975070485 namespace: kserve-ci-e2e-test resourceVersion: "27381" uid: 5e172598-db7e-437b-8f79-1a0abc5498de reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:20:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7f5a4-predictor namespace: kserve-ci-e2e-test resourceVersion: "26790" uid: 3ef6225f-e433-4888-b7ae-2b0b1072d058 kind: Event lastTimestamp: "2026-09-16T19:20:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:20:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:30Z" name: success-200-isvc-7f5a4-predictor.18d5e359757e9c05 namespace: kserve-ci-e2e-test resourceVersion: "27382" uid: 8757ab51-b4af-4e6f-9626-10696fc1696e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:19:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7f5a4 namespace: kserve-ci-e2e-test resourceVersion: "26386" uid: 46dfebd7-571f-4505-9cee-5dfe71577178 kind: Event lastTimestamp: "2026-09-16T19:20:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:19:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:20:03Z" name: success-200-isvc-7f5a4.18d5e34ef5113eba namespace: kserve-ci-e2e-test resourceVersion: "27078" uid: 9aa4b4d0-fce9-4640-8a3c-80338b8571a4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7f5a4 namespace: kserve-ci-e2e-test resourceVersion: "27075" uid: 46dfebd7-571f-4505-9cee-5dfe71577178 kind: Event lastTimestamp: "2026-09-16T19:20:03Z" message: InferenceService [success-200-isvc-7f5a4] is Ready metadata: creationTimestamp: "2026-09-16T19:20:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:20:03Z" name: success-200-isvc-7f5a4.18d5e35a24d7a4e9 namespace: kserve-ci-e2e-test resourceVersion: "27076" uid: 16723bcb-5766-4941-8a0a-303d6dcfe182 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:08:49.962710Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: 2e236bb2-a567-40ef-8a2b-dcaea42291c3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9450b-predictor-67ccb68c65-x62p8 to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:08:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:08:49Z" name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8.18d5e2bd6189444d namespace: kserve-ci-e2e-test resourceVersion: "20867" uid: 7dbd53fc-2ddc-4532-8a58-79087bbe8e29 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2e236bb2-a567-40ef-8a2b-dcaea42291c3 kind: Event lastTimestamp: "2026-09-16T19:08:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-9450b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:50Z" name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8.18d5e2bd6fc006b2 namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: b39f184c-6e92-4a58-99e8-c718e9495e47 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8 namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: 2e236bb2-a567-40ef-8a2b-dcaea42291c3 kind: Event lastTimestamp: "2026-09-16T19:08:51Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:08:51Z" name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8.18d5e2bda6e9b1f7 namespace: kserve-ci-e2e-test resourceVersion: "20969" uid: a686aa2f-97ce-4479-87aa-44ae56032e05 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2e236bb2-a567-40ef-8a2b-dcaea42291c3 kind: Event lastTimestamp: "2026-09-16T19:08:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:51Z" name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8.18d5e2bda88a2a62 namespace: kserve-ci-e2e-test resourceVersion: "20971" uid: 27bc2ccd-6d7e-482c-afef-8ae33a2ce4b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2e236bb2-a567-40ef-8a2b-dcaea42291c3 kind: Event lastTimestamp: "2026-09-16T19:08:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T19:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:51Z" name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8.18d5e2bdab5a2b3c namespace: kserve-ci-e2e-test resourceVersion: "20974" uid: 2fed1efd-160c-4e29-8b21-265b38674c78 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2e236bb2-a567-40ef-8a2b-dcaea42291c3 kind: Event lastTimestamp: "2026-09-16T19:08:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T19:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:51Z" name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8.18d5e2bdabc6abb1 namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: 21528d49-eca9-44a3-8d86-3f253230d1af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2e236bb2-a567-40ef-8a2b-dcaea42291c3 kind: Event lastTimestamp: "2026-09-16T19:08:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T19:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:51Z" name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8.18d5e2bdabcfd7d9 namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 20657d4f-d82d-4c38-a555-ca165019454b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2e236bb2-a567-40ef-8a2b-dcaea42291c3 kind: Event lastTimestamp: "2026-09-16T19:08:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T19:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:51Z" name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8.18d5e2bdae841c6d namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: 8b8dfe59-ea75-4599-ba37-3707059f56df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2e236bb2-a567-40ef-8a2b-dcaea42291c3 kind: Event lastTimestamp: "2026-09-16T19:08:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:51Z" name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8.18d5e2bdaef3f005 namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: 90abff0c-53b3-47c7-ae77-eba293f278f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:08:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2e236bb2-a567-40ef-8a2b-dcaea42291c3 kind: Event lastTimestamp: "2026-09-16T19:09:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T19:08:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:09:27Z" name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8.18d5e2bdd0697e9c namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: 34363ee0-1ae5-4611-afbe-6ee30d0edd6d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2e236bb2-a567-40ef-8a2b-dcaea42291c3 kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:00Z" name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8.18d5e2cdb6b14179 namespace: kserve-ci-e2e-test resourceVersion: "21796" uid: 9d453c12-4d7c-4f74-8991-684a1908e6ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2e236bb2-a567-40ef-8a2b-dcaea42291c3 kind: Event lastTimestamp: "2026-09-16T19:10:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:10:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:00Z" name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8.18d5e2cdb6b1cdf6 namespace: kserve-ci-e2e-test resourceVersion: "21801" uid: 4c037bb0-ea57-40b5-a17a-76867ef75fdc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:10:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 2e236bb2-a567-40ef-8a2b-dcaea42291c3 kind: Event lastTimestamp: "2026-09-16T19:10:03Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-09-16T19:10:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:10:03Z" name: success-200-isvc-9450b-predictor-67ccb68c65-x62p8.18d5e2ce94405f1c namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: 592ad4b9-95fc-4ff9-b7d9-ae90f9de39dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9450b-predictor-67ccb68c65 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: d3f7acc4-972d-498f-ad1f-22505c05bafb kind: Event lastTimestamp: "2026-09-16T19:08:49Z" message: 'Created pod: success-200-isvc-9450b-predictor-67ccb68c65-x62p8' metadata: creationTimestamp: "2026-09-16T19:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:49Z" name: success-200-isvc-9450b-predictor-67ccb68c65.18d5e2bd611d6217 namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 1b88e5bc-eff6-483e-92df-15ad919e9240 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9450b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 35f8f412-1fdc-4955-93b3-97bd4a7ffdbd kind: Event lastTimestamp: "2026-09-16T19:08:49Z" message: Scaled up replica set success-200-isvc-9450b-predictor-67ccb68c65 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:08:49Z" name: success-200-isvc-9450b-predictor.18d5e2bd5f2defe5 namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: c79820a0-802a-46fa-80f7-6834aa6cf97f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9450b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 6fb6146d-8ff6-4701-aed1-52f3d1b6073f kind: Event lastTimestamp: "2026-09-16T19:09:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:04Z" name: success-200-isvc-9450b-predictor.18d5e2c0ddb192bb namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: d851b048-ba4b-4169-bd64-de8eaa620a5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9450b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 6fb6146d-8ff6-4701-aed1-52f3d1b6073f kind: Event lastTimestamp: "2026-09-16T19:09:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:09:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:04Z" name: success-200-isvc-9450b-predictor.18d5e2c0de2ebd65 namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 3c37cdc7-dc03-4e02-8f1b-c742a8db5dde reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:09:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9450b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 6fb6146d-8ff6-4701-aed1-52f3d1b6073f kind: Event lastTimestamp: "2026-09-16T19:09:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:49Z" name: success-200-isvc-9450b-predictor.18d5e2c45d32b8e2 namespace: kserve-ci-e2e-test resourceVersion: "21670" uid: b8fb1d18-8547-45c9-b5f9-9c828e1236aa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:09:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9450b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 6fb6146d-8ff6-4701-aed1-52f3d1b6073f kind: Event lastTimestamp: "2026-09-16T19:09:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:09:49Z" name: success-200-isvc-9450b-predictor.18d5e2c45db30d7d namespace: kserve-ci-e2e-test resourceVersion: "21671" uid: b0d8bdf5-ccf2-49b4-8f59-e0fb6bb54698 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:08:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9450b namespace: kserve-ci-e2e-test resourceVersion: "20841" uid: e814d0e9-2a80-47f5-b9c1-efb661b92b36 kind: Event lastTimestamp: "2026-09-16T19:09:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T19:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:09:37Z" name: success-200-isvc-9450b.18d5e2bd5f329099 namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: 0725498b-ce57-472f-9f46-46b6c662be2b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:09:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9450b namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: e814d0e9-2a80-47f5-b9c1-efb661b92b36 kind: Event lastTimestamp: "2026-09-16T19:09:37Z" message: InferenceService [success-200-isvc-9450b] is Ready metadata: creationTimestamp: "2026-09-16T19:09:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:09:37Z" name: success-200-isvc-9450b.18d5e2c88c4b5c8d namespace: kserve-ci-e2e-test resourceVersion: "21535" uid: bffb419b-5aa4-4ecb-a589-ea288e01295e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:58:35.095904Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb namespace: kserve-ci-e2e-test resourceVersion: "15809" uid: a7d922f7-f4bb-4330-9503-8c5f15c244be kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T18:58:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:58:35Z" name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb.18d5e22e389e5b6c namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 7abc1b8c-631b-4c5e-9150-ff3199f74edf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: a7d922f7-f4bb-4330-9503-8c5f15c244be kind: Event lastTimestamp: "2026-09-16T18:58:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-9a5c2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T18:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:35Z" name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb.18d5e22e409124b2 namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: a54efd24-d4c6-47d5-9f25-66332d37de37 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb namespace: kserve-ci-e2e-test resourceVersion: "15818" uid: a7d922f7-f4bb-4330-9503-8c5f15c244be kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:58:36Z" name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb.18d5e22e7e38c3a3 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: 55a08972-ae1b-4eaa-9be3-4136f11f5bb1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: a7d922f7-f4bb-4330-9503-8c5f15c244be kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:36Z" name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb.18d5e22e7f9e4230 namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: 0f3a13e2-aabe-4f01-8023-94460db9191e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: a7d922f7-f4bb-4330-9503-8c5f15c244be kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:36Z" name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb.18d5e22e825934c5 namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: b597bdbf-583f-4e56-b177-31d302056b41 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: a7d922f7-f4bb-4330-9503-8c5f15c244be kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:36Z" name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb.18d5e22e82c9986f namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 9595a2c0-fffd-4486-8b94-50d34483e634 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: a7d922f7-f4bb-4330-9503-8c5f15c244be kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:36Z" name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb.18d5e22e82d35c3c namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 281e2c96-56a0-4f3b-9815-9d4c1a00ea78 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: a7d922f7-f4bb-4330-9503-8c5f15c244be kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:36Z" name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb.18d5e22e85b73dce namespace: kserve-ci-e2e-test resourceVersion: "15875" uid: fe6c606b-acc5-489b-ab4a-d8ace066ad53 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: a7d922f7-f4bb-4330-9503-8c5f15c244be kind: Event lastTimestamp: "2026-09-16T18:58:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:36Z" name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb.18d5e22e8629fa9e namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: ee765b51-e66a-4459-b1b3-dceeb1d1a84c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:58:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: a7d922f7-f4bb-4330-9503-8c5f15c244be kind: Event lastTimestamp: "2026-09-16T18:59:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:13Z" name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb.18d5e22ecb2c0cfb namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: dd64ccec-7b00-47bf-9d5d-5d9eebf78b64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: a7d922f7-f4bb-4330-9503-8c5f15c244be kind: Event lastTimestamp: "2026-09-16T19:07:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:50Z" name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb.18d5e2af759951f1 namespace: kserve-ci-e2e-test resourceVersion: "20248" uid: a800f20c-5ee4-44a1-a3b6-6e29c78ab1d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb namespace: kserve-ci-e2e-test resourceVersion: "15810" uid: a7d922f7-f4bb-4330-9503-8c5f15c244be kind: Event lastTimestamp: "2026-09-16T19:07:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:50Z" name: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb.18d5e2af759a7238 namespace: kserve-ci-e2e-test resourceVersion: "20250" uid: ba68a1fc-3706-4fff-949e-a58a4906b015 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9a5c2-predictor-79ccbb56df namespace: kserve-ci-e2e-test resourceVersion: "15802" uid: afff5b31-ecac-4999-87b3-0b5f6118e43d kind: Event lastTimestamp: "2026-09-16T18:58:35Z" message: 'Created pod: success-200-isvc-9a5c2-predictor-79ccbb56df-zsrwb' metadata: creationTimestamp: "2026-09-16T18:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:35Z" name: success-200-isvc-9a5c2-predictor-79ccbb56df.18d5e22e383a00e7 namespace: kserve-ci-e2e-test resourceVersion: "15812" uid: 16406a5d-4ed1-4b4b-89c3-fd0789f011f8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9a5c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15800" uid: e4ead618-7beb-41ce-80ff-96c6e7a7b337 kind: Event lastTimestamp: "2026-09-16T18:58:35Z" message: Scaled up replica set success-200-isvc-9a5c2-predictor-79ccbb56df from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:35Z" name: success-200-isvc-9a5c2-predictor.18d5e22e3634a629 namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: e62a012d-049c-47d5-9115-3df6cdd1696b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:58:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9a5c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: 563d2161-bc37-47af-ad0b-d19da170f2f4 kind: Event lastTimestamp: "2026-09-16T18:59:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:20Z" name: success-200-isvc-9a5c2-predictor.18d5e231b4b3fd0d namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: 79733c27-4800-4848-99c2-e4f5de949f21 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:58:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9a5c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: 563d2161-bc37-47af-ad0b-d19da170f2f4 kind: Event lastTimestamp: "2026-09-16T18:59:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:20Z" name: success-200-isvc-9a5c2-predictor.18d5e231b56f75b5 namespace: kserve-ci-e2e-test resourceVersion: "16334" uid: e9436528-92ef-4c59-ae26-c0e57983bd48 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:59:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9a5c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 563d2161-bc37-47af-ad0b-d19da170f2f4 kind: Event lastTimestamp: "2026-09-16T18:59:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:50Z" name: success-200-isvc-9a5c2-predictor.18d5e23534586213 namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: 5b35c02d-0737-405e-b6b2-c48a931fad0c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:59:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9a5c2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 563d2161-bc37-47af-ad0b-d19da170f2f4 kind: Event lastTimestamp: "2026-09-16T18:59:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:50Z" name: success-200-isvc-9a5c2-predictor.18d5e23534af0ac4 namespace: kserve-ci-e2e-test resourceVersion: "16842" uid: ab64bb82-8985-4c0c-b52c-93ef2f1a0785 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:58:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9a5c2 namespace: kserve-ci-e2e-test resourceVersion: "15788" uid: bb56836f-6eca-4639-af63-6998eaddac31 kind: Event lastTimestamp: "2026-09-16T18:59:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:59:23Z" name: success-200-isvc-9a5c2.18d5e22e3638ba4f namespace: kserve-ci-e2e-test resourceVersion: "16373" uid: 848763d5-2e9c-4d7d-af98-6bcff3de49e2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9a5c2 namespace: kserve-ci-e2e-test resourceVersion: "16371" uid: bb56836f-6eca-4639-af63-6998eaddac31 kind: Event lastTimestamp: "2026-09-16T18:59:23Z" message: InferenceService [success-200-isvc-9a5c2] is Ready metadata: creationTimestamp: "2026-09-16T18:59:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:59:23Z" name: success-200-isvc-9a5c2.18d5e239869507b0 namespace: kserve-ci-e2e-test resourceVersion: "16372" uid: 15508faa-b364-47ab-be86-81ae71b87f42 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:59:35.133553Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw namespace: kserve-ci-e2e-test resourceVersion: "16603" uid: 5c11b683-438e-445f-91f5-733cc6ff934a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a895e-predictor-79cfc466b9-vkzsw to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:59:35Z" name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw.18d5e23c332424e8 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: f820796a-1394-4fc6-ac77-4ebaddd1713a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: 5c11b683-438e-445f-91f5-733cc6ff934a kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:59:35Z" name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw.18d5e23c550a472d namespace: kserve-ci-e2e-test resourceVersion: "16667" uid: f5acfacb-2509-46fa-b58c-b6bffafa82b5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: 5c11b683-438e-445f-91f5-733cc6ff934a kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1962" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw.18d5e23c566b1196 namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: 75cb2110-d313-4ecc-8e1b-cca01bed9856 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: 5c11b683-438e-445f-91f5-733cc6ff934a kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw.18d5e23c590063c0 namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: 7c9b9c18-efd7-4c2a-a7ce-c08d4e8606f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: 5c11b683-438e-445f-91f5-733cc6ff934a kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw.18d5e23c597cff6c namespace: kserve-ci-e2e-test resourceVersion: "16674" uid: c4b82eee-cf66-476d-8a18-6ca7b8fab45c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: 5c11b683-438e-445f-91f5-733cc6ff934a kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw.18d5e23c5986f546 namespace: kserve-ci-e2e-test resourceVersion: "16676" uid: 8393b31a-ffa5-4639-a9dc-43af00352865 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: 5c11b683-438e-445f-91f5-733cc6ff934a kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw.18d5e23c5c220dd6 namespace: kserve-ci-e2e-test resourceVersion: "16681" uid: af0f8246-8fae-45fd-a13e-37270d0816d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: 5c11b683-438e-445f-91f5-733cc6ff934a kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw.18d5e23c5c8f33e8 namespace: kserve-ci-e2e-test resourceVersion: "16682" uid: 4c31c042-ed9b-49ff-ba3c-1ac5a61409d2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:59:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: 5c11b683-438e-445f-91f5-733cc6ff934a kind: Event lastTimestamp: "2026-09-16T19:00:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:00:13Z" name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw.18d5e23cd1211a55 namespace: kserve-ci-e2e-test resourceVersion: "17031" uid: 39426715-348f-420a-b878-03b78071b372 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: 5c11b683-438e-445f-91f5-733cc6ff934a kind: Event lastTimestamp: "2026-09-16T19:08:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T19:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:49Z" name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw.18d5e2bd594e8f1b namespace: kserve-ci-e2e-test resourceVersion: "20827" uid: fcf2b35b-afba-4920-8d8a-5ad4ca5944a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:08:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw namespace: kserve-ci-e2e-test resourceVersion: "16608" uid: 5c11b683-438e-445f-91f5-733cc6ff934a kind: Event lastTimestamp: "2026-09-16T19:08:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T19:08:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:49Z" name: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw.18d5e2bd594f1da0 namespace: kserve-ci-e2e-test resourceVersion: "20831" uid: 4a88a11f-fd97-444a-92cc-b0d02f7038c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a895e-predictor-79cfc466b9 namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 996bca09-cb50-4faf-99b0-759fbe066ada kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: 'Created pod: success-200-isvc-a895e-predictor-79cfc466b9-vkzsw' metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:35Z" name: success-200-isvc-a895e-predictor-79cfc466b9.18d5e23c32b9accf namespace: kserve-ci-e2e-test resourceVersion: "16607" uid: c8f6b6f0-189d-4937-9023-a563622d4ee4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a895e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: 443dc07c-c131-4ece-b527-8212d7519876 kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Scaled up replica set success-200-isvc-a895e-predictor-79cfc466b9 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:35Z" name: success-200-isvc-a895e-predictor.18d5e23c308892dc namespace: kserve-ci-e2e-test resourceVersion: "16591" uid: f7e723f8-9b80-4dfc-a1ec-5b596a1f42e8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:59:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a895e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: 9588033d-c5b6-42d2-bf8d-0cb0ac8decf8 kind: Event lastTimestamp: "2026-09-16T19:00:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:20Z" name: success-200-isvc-a895e-predictor.18d5e23fafd235fa namespace: kserve-ci-e2e-test resourceVersion: "17078" uid: 1577a4bf-a12e-4d95-9a84-fe7ee71ad733 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:59:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a895e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: 9588033d-c5b6-42d2-bf8d-0cb0ac8decf8 kind: Event lastTimestamp: "2026-09-16T19:00:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:20Z" name: success-200-isvc-a895e-predictor.18d5e23fb067e9ad namespace: kserve-ci-e2e-test resourceVersion: "17079" uid: f191d491-4da6-4b19-b9cf-4fadf923a609 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:00:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a895e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16836" uid: 9588033d-c5b6-42d2-bf8d-0cb0ac8decf8 kind: Event lastTimestamp: "2026-09-16T19:00:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:00:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:50Z" name: success-200-isvc-a895e-predictor.18d5e2432fc47628 namespace: kserve-ci-e2e-test resourceVersion: "17350" uid: 7630fcdf-e350-4a40-a6d4-37955b144d8b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T19:00:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a895e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16836" uid: 9588033d-c5b6-42d2-bf8d-0cb0ac8decf8 kind: Event lastTimestamp: "2026-09-16T19:00:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:00:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:50Z" name: success-200-isvc-a895e-predictor.18d5e243308d9152 namespace: kserve-ci-e2e-test resourceVersion: "17351" uid: db059146-3aa4-4eed-9aa7-d2b95666eb45 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a895e namespace: kserve-ci-e2e-test resourceVersion: "16570" uid: 295dcd47-b241-454a-9d38-a0f64923cbcc kind: Event lastTimestamp: "2026-09-16T19:00:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:00:23Z" name: success-200-isvc-a895e.18d5e23c30ad9a4c namespace: kserve-ci-e2e-test resourceVersion: "17123" uid: edf0ec1b-45c3-4f47-9d7a-f5078d172201 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:00:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a895e namespace: kserve-ci-e2e-test resourceVersion: "17121" uid: 295dcd47-b241-454a-9d38-a0f64923cbcc kind: Event lastTimestamp: "2026-09-16T19:00:23Z" message: InferenceService [success-200-isvc-a895e] is Ready metadata: creationTimestamp: "2026-09-16T19:00:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:00:23Z" name: success-200-isvc-a895e.18d5e2478d1a9538 namespace: kserve-ci-e2e-test resourceVersion: "17122" uid: 53f202c3-8f9f-48b1-9ae6-e528db6150b2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:57:04.959821Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14752" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-dc809-predictor-7df545f6f-cvr2x to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T18:57:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:57:04Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e2193c16db3b namespace: kserve-ci-e2e-test resourceVersion: "14756" uid: ce6f34d3-b0db-4c10-9319-272093bd56c1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14761" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: Add eth0 [10.132.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:57:05Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e2195e1774e6 namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: cc2d8b23-129f-49f4-a103-59aad4678ed4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: "2026-09-16T18:57:05Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1962" metadata: creationTimestamp: "2026-09-16T18:57:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:05Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e2195fae6b72 namespace: kserve-ci-e2e-test resourceVersion: "14854" uid: ef652bb8-a9b5-441c-bd83-f8aefaa429c9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: "2026-09-16T18:57:17Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1962" in 11.659s (11.659s including waiting). Image size: 1356756058 bytes.' metadata: creationTimestamp: "2026-09-16T18:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:17Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e21c16ab557e namespace: kserve-ci-e2e-test resourceVersion: "15001" uid: 7137338d-210e-4e16-b1d2-541d135eb114 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: "2026-09-16T18:57:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-16T18:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:17Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e21c1a82edb3 namespace: kserve-ci-e2e-test resourceVersion: "15003" uid: fd28cc6e-08a1-4552-9848-ba0fed1cf4b9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: "2026-09-16T18:57:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-16T18:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:17Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e21c1af197ca namespace: kserve-ci-e2e-test resourceVersion: "15004" uid: f0a89295-f7df-4234-9bd3-963feebea234 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: "2026-09-16T18:57:17Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-16T18:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:17Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e21c1afd4760 namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: 493c7be7-054c-4d9b-95d7-e33c4f9d24d9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: "2026-09-16T18:57:19Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.288s (2.288s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-16T18:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:19Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e21ca36357e2 namespace: kserve-ci-e2e-test resourceVersion: "15027" uid: 8463907c-4494-4cff-9639-75dadd084aa4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: "2026-09-16T18:57:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-16T18:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:19Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e21ca6900e8e namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: 517c864d-69b8-405a-9a6d-c4c4d0c45253 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: "2026-09-16T18:57:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:57:19Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e21ca7045a9d namespace: kserve-ci-e2e-test resourceVersion: "15031" uid: 8c4c9783-c183-4170-a9da-1237f83ff789 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-16T18:57:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: "2026-09-16T18:58:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.16:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:57:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:37Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e21d08ecbcfa namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: f2e1a32f-624f-4c56-be59-b8a33be5d104 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: "2026-09-16T18:58:34Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-16T18:58:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:34Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e22e3170f6cd namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 62a30ca6-3f9e-425c-948a-e8715c283090 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: "2026-09-16T18:58:34Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-16T18:58:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:34Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e22e31714bc6 namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: e7ced8c7-d75d-4a20-9d2f-44b964a634b7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: d1ac7ca3-a28b-44f6-981b-8e048fb5f02b kind: Event lastTimestamp: "2026-09-16T18:58:37Z" message: 'Readiness probe failed: Get "https://10.132.0.16:8643/healthz": dial tcp 10.132.0.16:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-16T18:58:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:37Z" name: success-200-isvc-dc809-predictor-7df545f6f-cvr2x.18d5e22ebb0d67c5 namespace: kserve-ci-e2e-test resourceVersion: "15897" uid: 8ebaec8c-d3fb-448a-95d8-a53b8ccb55b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-dc809-predictor-7df545f6f namespace: kserve-ci-e2e-test resourceVersion: "14744" uid: d05bf120-d3a1-4c2b-8b92-1ec6a941aefa kind: Event lastTimestamp: "2026-09-16T18:57:04Z" message: 'Created pod: success-200-isvc-dc809-predictor-7df545f6f-cvr2x' metadata: creationTimestamp: "2026-09-16T18:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:04Z" name: success-200-isvc-dc809-predictor-7df545f6f.18d5e2193ba66f96 namespace: kserve-ci-e2e-test resourceVersion: "14753" uid: 346fd070-1ef0-4004-b568-fb9d7067fdca reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:57:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-dc809-predictor namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: bf390522-5141-40b9-b16c-5e0a8878f082 kind: Event lastTimestamp: "2026-09-16T18:57:04Z" message: Scaled up replica set success-200-isvc-dc809-predictor-7df545f6f from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:04Z" name: success-200-isvc-dc809-predictor.18d5e2193935fa34 namespace: kserve-ci-e2e-test resourceVersion: "14745" uid: 41a0191d-1f99-4a2b-a613-4590cc23a0fb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:57:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dc809-predictor namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 3a6c94fe-a3e5-4088-8099-4ffebaa01527 kind: Event lastTimestamp: "2026-09-16T18:57:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:34Z" name: success-200-isvc-dc809-predictor.18d5e21cb7b1f4be namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: c3b5821a-a59a-4d09-9e8a-c4cb9f3e5e5a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-16T18:57:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dc809-predictor namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 3a6c94fe-a3e5-4088-8099-4ffebaa01527 kind: Event lastTimestamp: "2026-09-16T18:57:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:57:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:57:34Z" name: success-200-isvc-dc809-predictor.18d5e21cb8282861 namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: db61a0a5-90e7-4dd0-8205-fe30ff12e89c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:57:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dc809-predictor namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 3a6c94fe-a3e5-4088-8099-4ffebaa01527 kind: Event lastTimestamp: "2026-09-16T18:58:19Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:19Z" name: success-200-isvc-dc809-predictor.18d5e223b5df5405 namespace: kserve-ci-e2e-test resourceVersion: "15580" uid: 2914ddbb-0f08-4ea8-86b1-266549623b16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:57:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dc809-predictor namespace: kserve-ci-e2e-test resourceVersion: "15036" uid: 3a6c94fe-a3e5-4088-8099-4ffebaa01527 kind: Event lastTimestamp: "2026-09-16T18:58:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T18:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:19Z" name: success-200-isvc-dc809-predictor.18d5e223b63329da namespace: kserve-ci-e2e-test resourceVersion: "15581" uid: ed06468a-b634-47a6-9329-8a720cc41256 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-16T18:57:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dc809 namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 34fd86fe-696b-4bdb-bb29-480516cabae9 kind: Event lastTimestamp: "2026-09-16T18:58:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-16T18:57:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:07Z" name: success-200-isvc-dc809.18d5e219393a7d41 namespace: kserve-ci-e2e-test resourceVersion: "15480" uid: 61f1f17f-61cd-4659-8f66-fc9eb9db1a46 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dc809 namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: 34fd86fe-696b-4bdb-bb29-480516cabae9 kind: Event lastTimestamp: "2026-09-16T18:58:07Z" message: InferenceService [success-200-isvc-dc809] is Ready metadata: creationTimestamp: "2026-09-16T18:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:07Z" name: success-200-isvc-dc809.18d5e227c4e2d94e namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: a9b14c23-17e5-4b6c-be38-540a8847a477 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-16T19:20:15.237661Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7f5a4-6d85f7cc45-dh59j namespace: kserve-ci-e2e-test resourceVersion: "27207" uid: 0d7f486c-5298-4b04-963d-f6de588ca1f2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-7f5a4-6d85f7cc45-dh59j to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T19:20:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T19:20:15Z" name: switch-graph-7f5a4-6d85f7cc45-dh59j.18d5e35cef1b24e4 namespace: kserve-ci-e2e-test resourceVersion: "27212" uid: 197624a3-7a0a-449c-9b48-e60cdcd0c6f2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:15Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7f5a4-6d85f7cc45-dh59j namespace: kserve-ci-e2e-test resourceVersion: "27216" uid: 0d7f486c-5298-4b04-963d-f6de588ca1f2 kind: Event lastTimestamp: "2026-09-16T19:20:15Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T19:20:15Z" name: switch-graph-7f5a4-6d85f7cc45-dh59j.18d5e35d1140a90c namespace: kserve-ci-e2e-test resourceVersion: "27234" uid: ab58709c-9369-4844-af69-586d164c72ea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f5a4} kind: Pod name: switch-graph-7f5a4-6d85f7cc45-dh59j namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 0d7f486c-5298-4b04-963d-f6de588ca1f2 kind: Event lastTimestamp: "2026-09-16T19:20:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:83a04242aef3bc251b4da1ba3ce544c38d7df9880cbb933157aae8432a36f471" already present on machine metadata: creationTimestamp: "2026-09-16T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:20:15Z" name: switch-graph-7f5a4-6d85f7cc45-dh59j.18d5e35d12887773 namespace: kserve-ci-e2e-test resourceVersion: "27236" uid: 9cd6cbae-4e3e-4f17-807e-e0daf96e25a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f5a4} kind: Pod name: switch-graph-7f5a4-6d85f7cc45-dh59j namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 0d7f486c-5298-4b04-963d-f6de588ca1f2 kind: Event lastTimestamp: "2026-09-16T19:20:15Z" message: 'Created container: switch-graph-7f5a4' metadata: creationTimestamp: "2026-09-16T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:20:15Z" name: switch-graph-7f5a4-6d85f7cc45-dh59j.18d5e35d152b9d31 namespace: kserve-ci-e2e-test resourceVersion: "27237" uid: 6fe29cf0-3ab2-4694-9a41-5ef942d383ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f5a4} kind: Pod name: switch-graph-7f5a4-6d85f7cc45-dh59j namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 0d7f486c-5298-4b04-963d-f6de588ca1f2 kind: Event lastTimestamp: "2026-09-16T19:20:15Z" message: Started container switch-graph-7f5a4 metadata: creationTimestamp: "2026-09-16T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:20:15Z" name: switch-graph-7f5a4-6d85f7cc45-dh59j.18d5e35d15998db0 namespace: kserve-ci-e2e-test resourceVersion: "27238" uid: a331f370-1bf0-4b12-a5e1-5db657adcdd2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:36:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f5a4} kind: Pod name: switch-graph-7f5a4-6d85f7cc45-dh59j namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 0d7f486c-5298-4b04-963d-f6de588ca1f2 kind: Event lastTimestamp: "2026-09-16T19:36:34Z" message: Stopping container switch-graph-7f5a4 metadata: creationTimestamp: "2026-09-16T19:36:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:36:34Z" name: switch-graph-7f5a4-6d85f7cc45-dh59j.18d5e440e0399ff4 namespace: kserve-ci-e2e-test resourceVersion: "34046" uid: 1ff79366-8486-49e6-addf-85f523d98ab5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:36:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7f5a4} kind: Pod name: switch-graph-7f5a4-6d85f7cc45-dh59j namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 0d7f486c-5298-4b04-963d-f6de588ca1f2 kind: Event lastTimestamp: "2026-09-16T19:37:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:36:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:37:02Z" name: switch-graph-7f5a4-6d85f7cc45-dh59j.18d5e4419c08b1d4 namespace: kserve-ci-e2e-test resourceVersion: "34326" uid: 06cb28f2-f40d-47f3-ab88-15bed81b0a16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-7f5a4-6d85f7cc45 namespace: kserve-ci-e2e-test resourceVersion: "27204" uid: 9986bdd0-c62e-4835-81cc-169706c4a34f kind: Event lastTimestamp: "2026-09-16T19:20:15Z" message: 'Created pod: switch-graph-7f5a4-6d85f7cc45-dh59j' metadata: creationTimestamp: "2026-09-16T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:15Z" name: switch-graph-7f5a4-6d85f7cc45.18d5e35ceebc0247 namespace: kserve-ci-e2e-test resourceVersion: "27208" uid: 6166fb71-2471-4f0a-a9f1-24f36b3a93fb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-7f5a4 namespace: kserve-ci-e2e-test resourceVersion: "27202" uid: faed5cdc-3083-492e-b569-66c2b55af891 kind: Event lastTimestamp: "2026-09-16T19:20:15Z" message: Scaled up replica set switch-graph-7f5a4-6d85f7cc45 from 0 to 1 metadata: creationTimestamp: "2026-09-16T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:15Z" name: switch-graph-7f5a4.18d5e35cedd516f4 namespace: kserve-ci-e2e-test resourceVersion: "27205" uid: 9bb2d36d-3f25-4661-b048-6ab9dd64e5e2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-7f5a4 namespace: kserve-ci-e2e-test resourceVersion: "27219" uid: 19b97583-d7a4-4c73-8709-1df28035e175 kind: Event lastTimestamp: "2026-09-16T19:20:15Z" message: InferenceGraph [switch-graph-7f5a4] is Ready metadata: creationTimestamp: "2026-09-16T19:20:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T19:20:15Z" name: switch-graph-7f5a4.18d5e35cf44d2a10 namespace: kserve-ci-e2e-test resourceVersion: "27223" uid: 70b7e65f-a2b1-4b12-a788-10b2efe00f57 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7f5a4 namespace: kserve-ci-e2e-test resourceVersion: "27203" uid: b0bc0b44-fadf-4ea4-85e6-94e95a7efb9d kind: Event lastTimestamp: "2026-09-16T19:20:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:30Z" name: switch-graph-7f5a4.18d5e3606d406916 namespace: kserve-ci-e2e-test resourceVersion: "27379" uid: d8207641-4966-40ba-9727-01714cec8c73 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7f5a4 namespace: kserve-ci-e2e-test resourceVersion: "27203" uid: b0bc0b44-fadf-4ea4-85e6-94e95a7efb9d kind: Event lastTimestamp: "2026-09-16T19:20:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T19:20:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:30Z" name: switch-graph-7f5a4.18d5e3606de4c7f0 namespace: kserve-ci-e2e-test resourceVersion: "27380" uid: 4a4ee988-7516-46bd-b662-9565072919e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7f5a4 namespace: kserve-ci-e2e-test resourceVersion: "27378" uid: b0bc0b44-fadf-4ea4-85e6-94e95a7efb9d kind: Event lastTimestamp: "2026-09-16T19:20:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:45Z" name: switch-graph-7f5a4.18d5e363ed114aa3 namespace: kserve-ci-e2e-test resourceVersion: "27503" uid: f2a1235f-a2c1-4512-be2d-d5577c5f97c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:20:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7f5a4 namespace: kserve-ci-e2e-test resourceVersion: "27378" uid: b0bc0b44-fadf-4ea4-85e6-94e95a7efb9d kind: Event lastTimestamp: "2026-09-16T19:20:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-16T19:20:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:20:45Z" name: switch-graph-7f5a4.18d5e363ed74bd32 namespace: kserve-ci-e2e-test resourceVersion: "27505" uid: 9b90c51e-916f-4494-bf3e-9347caf4507d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:59:35.113927Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9a5c2-755f799d8b-t59th namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: f749f10b-5983-4dde-ae73-d31c25d8051e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-9a5c2-755f799d8b-t59th to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:59:35Z" name: switch-graph-9a5c2-755f799d8b-t59th.18d5e23c31f89ba2 namespace: kserve-ci-e2e-test resourceVersion: "16604" uid: c94a9790-7c22-4555-945d-68d450520e1f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9a5c2-755f799d8b-t59th namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: f749f10b-5983-4dde-ae73-d31c25d8051e kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:59:35Z" name: switch-graph-9a5c2-755f799d8b-t59th.18d5e23c53e5b2b1 namespace: kserve-ci-e2e-test resourceVersion: "16665" uid: 88064f3a-15f3-496a-a5fc-a61511917315 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9a5c2} kind: Pod name: switch-graph-9a5c2-755f799d8b-t59th namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: f749f10b-5983-4dde-ae73-d31c25d8051e kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:83a04242aef3bc251b4da1ba3ce544c38d7df9880cbb933157aae8432a36f471" already present on machine metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: switch-graph-9a5c2-755f799d8b-t59th.18d5e23c553cff74 namespace: kserve-ci-e2e-test resourceVersion: "16668" uid: 121fdfa0-fc28-4646-928a-0f41228d8cd6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9a5c2} kind: Pod name: switch-graph-9a5c2-755f799d8b-t59th namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: f749f10b-5983-4dde-ae73-d31c25d8051e kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: 'Created container: switch-graph-9a5c2' metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: switch-graph-9a5c2-755f799d8b-t59th.18d5e23c58824a85 namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: d8038492-460a-4088-8e57-506d75c3b8fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9a5c2} kind: Pod name: switch-graph-9a5c2-755f799d8b-t59th namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: f749f10b-5983-4dde-ae73-d31c25d8051e kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Started container switch-graph-9a5c2 metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:35Z" name: switch-graph-9a5c2-755f799d8b-t59th.18d5e23c58f18867 namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: 7c0ae506-6032-4039-bfe1-1d2cbf106690 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T19:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9a5c2} kind: Pod name: switch-graph-9a5c2-755f799d8b-t59th namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: f749f10b-5983-4dde-ae73-d31c25d8051e kind: Event lastTimestamp: "2026-09-16T19:07:49Z" message: Stopping container switch-graph-9a5c2 metadata: creationTimestamp: "2026-09-16T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:07:49Z" name: switch-graph-9a5c2-755f799d8b-t59th.18d5e2af60b83050 namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: 55db2e12-5706-4fa2-b9fc-7eba829fb14c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9a5c2} kind: Pod name: switch-graph-9a5c2-755f799d8b-t59th namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: f749f10b-5983-4dde-ae73-d31c25d8051e kind: Event lastTimestamp: "2026-09-16T19:08:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T19:08:16Z" name: switch-graph-9a5c2-755f799d8b-t59th.18d5e2afd5c2d426 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 01f09ee1-1eed-4ef9-abcf-ed211e926f1e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-9a5c2-755f799d8b namespace: kserve-ci-e2e-test resourceVersion: "16590" uid: d465346c-0816-47ea-b54c-89db0e1892cc kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: 'Created pod: switch-graph-9a5c2-755f799d8b-t59th' metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:35Z" name: switch-graph-9a5c2-755f799d8b.18d5e23c31876489 namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: 4772c917-24bc-4cca-9c40-00fd558a118c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-9a5c2 namespace: kserve-ci-e2e-test resourceVersion: "16586" uid: fdea2b68-a65f-4adf-8364-77dd342b63f8 kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: Scaled up replica set switch-graph-9a5c2-755f799d8b from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:59:35Z" name: switch-graph-9a5c2.18d5e23c30ba31cf namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: c38d49bc-dc4b-43d2-b451-10dfb53fdea9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:59:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-9a5c2 namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: 493c520b-ee79-4e4e-8689-11588d9cadd2 kind: Event lastTimestamp: "2026-09-16T18:59:35Z" message: InferenceGraph [switch-graph-9a5c2] is Ready metadata: creationTimestamp: "2026-09-16T18:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:59:35Z" name: switch-graph-9a5c2.18d5e23c37521472 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 8d57757b-8d66-40ce-aaa1-f215aa2ec657 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:59:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9a5c2 namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: 4e854208-1b27-46ce-9fba-8bccd7ff4bfd kind: Event lastTimestamp: "2026-09-16T19:00:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:20Z" name: switch-graph-9a5c2.18d5e23fb048deb8 namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: adb543d7-df1d-4c7d-aeeb-a2cc75ef7aea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-16T18:59:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9a5c2 namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: 4e854208-1b27-46ce-9fba-8bccd7ff4bfd kind: Event lastTimestamp: "2026-09-16T19:00:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-16T18:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T19:00:20Z" name: switch-graph-9a5c2.18d5e23fb0c16837 namespace: kserve-ci-e2e-test resourceVersion: "17076" uid: cae875ac-b0a1-421f-a82f-cdce6bf48c72 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-16T18:58:24.831757Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-dc809-8f5cc9c67-kfk4h namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 63d08ebb-7f01-4fe1-b1f7-6e8b1ae6ca9c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-dc809-8f5cc9c67-kfk4h to ip-10-0-137-226.ec2.internal metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-16T18:58:24Z" name: switch-graph-dc809-8f5cc9c67-kfk4h.18d5e22bd4d3cf81 namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 768114a1-e6ae-4370-904e-c6a51208e8bc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855f8cb756-vkbbv source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:24Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-dc809-8f5cc9c67-kfk4h namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 63d08ebb-7f01-4fe1-b1f7-6e8b1ae6ca9c kind: Event lastTimestamp: "2026-09-16T18:58:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-dc809-serving-cert" not found' metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:24Z" name: switch-graph-dc809-8f5cc9c67-kfk4h.18d5e22bdc8bb60f namespace: kserve-ci-e2e-test resourceVersion: "15646" uid: 7b00b420-d296-46e3-a55b-3dba77df0a5a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:25Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-dc809-8f5cc9c67-kfk4h namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: 63d08ebb-7f01-4fe1-b1f7-6e8b1ae6ca9c kind: Event lastTimestamp: "2026-09-16T18:58:25Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-16T18:58:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-16T18:58:25Z" name: switch-graph-dc809-8f5cc9c67-kfk4h.18d5e22c1a5aa40b namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: fd81bb36-a6be-48f7-b933-11cde5b0c6cf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dc809} kind: Pod name: switch-graph-dc809-8f5cc9c67-kfk4h namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 63d08ebb-7f01-4fe1-b1f7-6e8b1ae6ca9c kind: Event lastTimestamp: "2026-09-16T18:58:26Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:83a04242aef3bc251b4da1ba3ce544c38d7df9880cbb933157aae8432a36f471" metadata: creationTimestamp: "2026-09-16T18:58:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:26Z" name: switch-graph-dc809-8f5cc9c67-kfk4h.18d5e22c1bb72abd namespace: kserve-ci-e2e-test resourceVersion: "15659" uid: dcadcf12-1c24-4795-8572-8ae9acd5a101 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dc809} kind: Pod name: switch-graph-dc809-8f5cc9c67-kfk4h namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 63d08ebb-7f01-4fe1-b1f7-6e8b1ae6ca9c kind: Event lastTimestamp: "2026-09-16T18:58:28Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:83a04242aef3bc251b4da1ba3ce544c38d7df9880cbb933157aae8432a36f471" in 2.377s (2.377s including waiting). Image size: 211458663 bytes.' metadata: creationTimestamp: "2026-09-16T18:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:28Z" name: switch-graph-dc809-8f5cc9c67-kfk4h.18d5e22ca96e93b4 namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 8a78486e-a7ee-43c1-8387-27a234dfc8e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dc809} kind: Pod name: switch-graph-dc809-8f5cc9c67-kfk4h namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 63d08ebb-7f01-4fe1-b1f7-6e8b1ae6ca9c kind: Event lastTimestamp: "2026-09-16T18:58:28Z" message: 'Created container: switch-graph-dc809' metadata: creationTimestamp: "2026-09-16T18:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:28Z" name: switch-graph-dc809-8f5cc9c67-kfk4h.18d5e22cac4c280c namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: e3077b6e-fc8c-45a1-a4d7-5c2bf2c6282b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dc809} kind: Pod name: switch-graph-dc809-8f5cc9c67-kfk4h namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 63d08ebb-7f01-4fe1-b1f7-6e8b1ae6ca9c kind: Event lastTimestamp: "2026-09-16T18:58:28Z" message: Started container switch-graph-dc809 metadata: creationTimestamp: "2026-09-16T18:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:28Z" name: switch-graph-dc809-8f5cc9c67-kfk4h.18d5e22cacab57b1 namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: ba4c9626-2af3-45cd-93aa-d5ad9e4c0e83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dc809} kind: Pod name: switch-graph-dc809-8f5cc9c67-kfk4h namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 63d08ebb-7f01-4fe1-b1f7-6e8b1ae6ca9c kind: Event lastTimestamp: "2026-09-16T18:58:34Z" message: Stopping container switch-graph-dc809 metadata: creationTimestamp: "2026-09-16T18:58:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:58:34Z" name: switch-graph-dc809-8f5cc9c67-kfk4h.18d5e22e2c52f02d namespace: kserve-ci-e2e-test resourceVersion: "15753" uid: f476444b-f76a-4ecd-8027-96b3a2adc757 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-16T18:58:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dc809} kind: Pod name: switch-graph-dc809-8f5cc9c67-kfk4h namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 63d08ebb-7f01-4fe1-b1f7-6e8b1ae6ca9c kind: Event lastTimestamp: "2026-09-16T18:59:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-16T18:58:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-16T18:59:04Z" name: switch-graph-dc809-8f5cc9c67-kfk4h.18d5e22f4069cad2 namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: 9f892891-a7c7-4ea3-a7ef-c831580072ff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-226.ec2.internal source: component: kubelet host: ip-10-0-137-226.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-dc809-8f5cc9c67 namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: d499bc7d-2184-448c-acd8-d777c896240d kind: Event lastTimestamp: "2026-09-16T18:58:24Z" message: 'Created pod: switch-graph-dc809-8f5cc9c67-kfk4h' metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:24Z" name: switch-graph-dc809-8f5cc9c67.18d5e22bd41f9ba8 namespace: kserve-ci-e2e-test resourceVersion: "15632" uid: 1225713e-ddc4-4623-b25e-9c02f4bb2a2b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-dc809 namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 3df5980b-fa73-4458-a220-2e77ceaa8c2a kind: Event lastTimestamp: "2026-09-16T18:58:24Z" message: Scaled up replica set switch-graph-dc809-8f5cc9c67 from 0 to 1 metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-16T18:58:24Z" name: switch-graph-dc809.18d5e22bd247d6a4 namespace: kserve-ci-e2e-test resourceVersion: "15629" uid: d1655e9e-eb55-41ee-b1a4-3d60b00a9a51 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-16T18:58:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-dc809 namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: 3c6d2624-a915-47fb-b700-b60bc6e9b6bd kind: Event lastTimestamp: "2026-09-16T18:58:24Z" message: InferenceGraph [switch-graph-dc809] is Ready metadata: creationTimestamp: "2026-09-16T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-16T18:58:24Z" name: switch-graph-dc809.18d5e22bd88940d2 namespace: kserve-ci-e2e-test resourceVersion: "15644" uid: 06976aea-586d-4dc9-8dc5-0ea8ccf0e101 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "34339"