--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:12:05.366725Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-06a6c-784596677c-tv66c namespace: kserve-ci-e2e-test resourceVersion: "22160" uid: d823ad5c-3459-4a8e-857d-72d68773b2bc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-06a6c-784596677c-tv66c to ip-10-0-136-197.ec2.internal metadata: creationTimestamp: "2026-07-31T15:12:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:12:05Z" name: ensemble-graph-06a6c-784596677c-tv66c.18c7688b70225f5b namespace: kserve-ci-e2e-test resourceVersion: "22165" uid: c1221c08-2a79-4928-8bca-4692a82b4f33 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:05Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-06a6c-784596677c-tv66c namespace: kserve-ci-e2e-test resourceVersion: "22169" uid: d823ad5c-3459-4a8e-857d-72d68773b2bc kind: Event lastTimestamp: "2026-07-31T15:12:05Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:12:05Z" name: ensemble-graph-06a6c-784596677c-tv66c.18c7688b89d40514 namespace: kserve-ci-e2e-test resourceVersion: "22182" uid: 6077fe9b-8c2e-4f7f-8b19-38fdf797070a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-06a6c} kind: Pod name: ensemble-graph-06a6c-784596677c-tv66c namespace: kserve-ci-e2e-test resourceVersion: "22163" uid: d823ad5c-3459-4a8e-857d-72d68773b2bc kind: Event lastTimestamp: "2026-07-31T15:12:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:45f424b4a00ee86756776b91e414e15d8d34d9dfbb27b1d7b22dfeb283636eda" already present on machine metadata: creationTimestamp: "2026-07-31T15:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:05Z" name: ensemble-graph-06a6c-784596677c-tv66c.18c7688b8b053f49 namespace: kserve-ci-e2e-test resourceVersion: "22184" uid: 526154f2-3683-45d8-a299-06091787e05c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-06a6c} kind: Pod name: ensemble-graph-06a6c-784596677c-tv66c namespace: kserve-ci-e2e-test resourceVersion: "22163" uid: d823ad5c-3459-4a8e-857d-72d68773b2bc kind: Event lastTimestamp: "2026-07-31T15:12:05Z" message: 'Created container: ensemble-graph-06a6c' metadata: creationTimestamp: "2026-07-31T15:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:05Z" name: ensemble-graph-06a6c-784596677c-tv66c.18c7688b8e7eb82f namespace: kserve-ci-e2e-test resourceVersion: "22185" uid: 4229ccc4-2ab8-482d-8301-e1da7ca86ba3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-06a6c} kind: Pod name: ensemble-graph-06a6c-784596677c-tv66c namespace: kserve-ci-e2e-test resourceVersion: "22163" uid: d823ad5c-3459-4a8e-857d-72d68773b2bc kind: Event lastTimestamp: "2026-07-31T15:12:05Z" message: Started container ensemble-graph-06a6c metadata: creationTimestamp: "2026-07-31T15:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:05Z" name: ensemble-graph-06a6c-784596677c-tv66c.18c7688b8ef861fd namespace: kserve-ci-e2e-test resourceVersion: "22186" uid: 362b0709-a77c-4e66-9a2c-96884fd6f9db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-06a6c} kind: Pod name: ensemble-graph-06a6c-784596677c-tv66c namespace: kserve-ci-e2e-test resourceVersion: "22163" uid: d823ad5c-3459-4a8e-857d-72d68773b2bc kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: Stopping container ensemble-graph-06a6c metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:20Z" name: ensemble-graph-06a6c-784596677c-tv66c.18c768fe9ba0bfd3 namespace: kserve-ci-e2e-test resourceVersion: "26140" uid: 5acc1139-99fb-45e0-bfa3-9c53c52d6dc7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:20:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-06a6c} kind: Pod name: ensemble-graph-06a6c-784596677c-tv66c namespace: kserve-ci-e2e-test resourceVersion: "22163" uid: d823ad5c-3459-4a8e-857d-72d68773b2bc kind: Event lastTimestamp: "2026-07-31T15:20:47Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-31T15:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:47Z" name: ensemble-graph-06a6c-784596677c-tv66c.18c768ff354f2f12 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: de84d5de-a4de-4e3c-ae18-9296b296d8eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-06a6c-784596677c namespace: kserve-ci-e2e-test resourceVersion: "22157" uid: 99c517e8-ad8c-4975-82dc-cc5c2dcd89bc kind: Event lastTimestamp: "2026-07-31T15:12:05Z" message: 'Created pod: ensemble-graph-06a6c-784596677c-tv66c' metadata: creationTimestamp: "2026-07-31T15:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:05Z" name: ensemble-graph-06a6c-784596677c.18c7688b6fb8df9b namespace: kserve-ci-e2e-test resourceVersion: "22161" uid: bd57c629-b064-451e-807c-11f112a49549 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-06a6c namespace: kserve-ci-e2e-test resourceVersion: "22152" uid: 11a7c564-a9c7-4f80-b84d-e42d7569d4c8 kind: Event lastTimestamp: "2026-07-31T15:12:05Z" message: Scaled up replica set ensemble-graph-06a6c-784596677c from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:05Z" name: ensemble-graph-06a6c.18c7688b6f26817a namespace: kserve-ci-e2e-test resourceVersion: "22158" uid: b17e8ccf-12a0-4084-afad-95834a5cd166 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:05Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-06a6c namespace: kserve-ci-e2e-test resourceVersion: "22171" uid: fb0ee954-9f53-43e8-ac3a-46a43eb427ed kind: Event lastTimestamp: "2026-07-31T15:12:05Z" message: InferenceGraph [ensemble-graph-06a6c] is Ready metadata: creationTimestamp: "2026-07-31T15:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:12:05Z" name: ensemble-graph-06a6c.18c7688b732350b0 namespace: kserve-ci-e2e-test resourceVersion: "22173" uid: d7d23bac-3040-4a76-9d3b-93c492033e95 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:12:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-06a6c namespace: kserve-ci-e2e-test resourceVersion: "22155" uid: 4e0302f4-9cfe-44a8-85d4-a19959bae9a7 kind: Event lastTimestamp: "2026-07-31T15:12:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:12:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:35Z" name: ensemble-graph-06a6c.18c7688eedba42a6 namespace: kserve-ci-e2e-test resourceVersion: "22577" uid: b5748b50-7d54-4ab1-a357-f8caa07d067c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:12:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-06a6c namespace: kserve-ci-e2e-test resourceVersion: "22155" uid: 4e0302f4-9cfe-44a8-85d4-a19959bae9a7 kind: Event lastTimestamp: "2026-07-31T15:12:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:12:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:35Z" name: ensemble-graph-06a6c.18c7688eee19e393 namespace: kserve-ci-e2e-test resourceVersion: "22578" uid: 06a9264a-1dcb-4859-82d5-c1f8db4926a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:10:55.041640Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-49957-646f5768d6-8lwz6 namespace: kserve-ci-e2e-test resourceVersion: "21338" uid: 0aa88c76-2edb-4cef-96b8-cb6ec3bcbf3d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-49957-646f5768d6-8lwz6 to ip-10-0-136-197.ec2.internal metadata: creationTimestamp: "2026-07-31T15:10:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:10:55Z" name: ensemble-graph-49957-646f5768d6-8lwz6.18c7687b106ebfd8 namespace: kserve-ci-e2e-test resourceVersion: "21343" uid: ca509729-06cf-416c-85cc-ed71c53b4533 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:55Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-49957-646f5768d6-8lwz6 namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 0aa88c76-2edb-4cef-96b8-cb6ec3bcbf3d kind: Event lastTimestamp: "2026-07-31T15:10:55Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:10:55Z" name: ensemble-graph-49957-646f5768d6-8lwz6.18c7687b2a17c108 namespace: kserve-ci-e2e-test resourceVersion: "21360" uid: 9eed4766-c1e6-449e-9295-0db4133a19f6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-49957} kind: Pod name: ensemble-graph-49957-646f5768d6-8lwz6 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 0aa88c76-2edb-4cef-96b8-cb6ec3bcbf3d kind: Event lastTimestamp: "2026-07-31T15:10:55Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:45f424b4a00ee86756776b91e414e15d8d34d9dfbb27b1d7b22dfeb283636eda" already present on machine metadata: creationTimestamp: "2026-07-31T15:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:55Z" name: ensemble-graph-49957-646f5768d6-8lwz6.18c7687b2b6b0b85 namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: 4f4d94c9-4688-4c0f-8e8f-147fafd9b646 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-49957} kind: Pod name: ensemble-graph-49957-646f5768d6-8lwz6 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 0aa88c76-2edb-4cef-96b8-cb6ec3bcbf3d kind: Event lastTimestamp: "2026-07-31T15:10:55Z" message: 'Created container: ensemble-graph-49957' metadata: creationTimestamp: "2026-07-31T15:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:55Z" name: ensemble-graph-49957-646f5768d6-8lwz6.18c7687b2eff9a70 namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: c274d170-b29f-4b29-9cc6-f5b3b06b7419 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-49957} kind: Pod name: ensemble-graph-49957-646f5768d6-8lwz6 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 0aa88c76-2edb-4cef-96b8-cb6ec3bcbf3d kind: Event lastTimestamp: "2026-07-31T15:10:55Z" message: Started container ensemble-graph-49957 metadata: creationTimestamp: "2026-07-31T15:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:55Z" name: ensemble-graph-49957-646f5768d6-8lwz6.18c7687b2f794c47 namespace: kserve-ci-e2e-test resourceVersion: "21366" uid: bb440983-e5db-43e7-8fd3-c76b46483ed7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-49957} kind: Pod name: ensemble-graph-49957-646f5768d6-8lwz6 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 0aa88c76-2edb-4cef-96b8-cb6ec3bcbf3d kind: Event lastTimestamp: "2026-07-31T15:11:05Z" message: Stopping container ensemble-graph-49957 metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:05Z" name: ensemble-graph-49957-646f5768d6-8lwz6.18c7687d69c2a5c0 namespace: kserve-ci-e2e-test resourceVersion: "21465" uid: d61d3ee8-3718-43eb-91d4-5802ac9456ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-49957} kind: Pod name: ensemble-graph-49957-646f5768d6-8lwz6 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 0aa88c76-2edb-4cef-96b8-cb6ec3bcbf3d kind: Event lastTimestamp: "2026-07-31T15:11:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-31T15:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:32Z" name: ensemble-graph-49957-646f5768d6-8lwz6.18c7687de9293067 namespace: kserve-ci-e2e-test resourceVersion: "21823" uid: 1f4b7d3a-1009-4b54-8ed1-76266e4c7be8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-49957-646f5768d6 namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: 3dbe9e0b-ad4e-45cc-8fb6-a14aa221fef7 kind: Event lastTimestamp: "2026-07-31T15:10:55Z" message: 'Created pod: ensemble-graph-49957-646f5768d6-8lwz6' metadata: creationTimestamp: "2026-07-31T15:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:55Z" name: ensemble-graph-49957-646f5768d6.18c7687b0fec0b89 namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: 019c7cbd-9579-41fb-82b9-46535bdebdd2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-49957 namespace: kserve-ci-e2e-test resourceVersion: "21332" uid: 7e5d05e1-c806-4780-8fa0-6eb37c05b6a0 kind: Event lastTimestamp: "2026-07-31T15:10:55Z" message: Scaled up replica set ensemble-graph-49957-646f5768d6 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:55Z" name: ensemble-graph-49957.18c7687b0f2c1e30 namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: 4078ba66-f256-49b3-93a0-bb38b4e0ad3a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-49957 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 5be1ba36-bbb1-49b3-85ec-64ecab293963 kind: Event lastTimestamp: "2026-07-31T15:10:55Z" message: InferenceGraph [ensemble-graph-49957] is Ready metadata: creationTimestamp: "2026-07-31T15:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:10:55Z" name: ensemble-graph-49957.18c7687b14c2ea43 namespace: kserve-ci-e2e-test resourceVersion: "21355" uid: aa42b1c2-d2f5-4044-b742-5a90567b96a1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:11:05.441410Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g namespace: kserve-ci-e2e-test resourceVersion: "21543" uid: e3a6010f-eee8-4a8c-894e-d5b21bec7a31 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:11:05Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g.18c7687d7c4e8c48 namespace: kserve-ci-e2e-test resourceVersion: "21547" uid: 053c096c-64be-49d9-a685-80c5de165884 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: e3a6010f-eee8-4a8c-894e-d5b21bec7a31 kind: Event lastTimestamp: "2026-07-31T15:11:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-06a6c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:05Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g.18c7687d845d7545 namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: b8e340e1-4c4a-4fe2-9fb8-eb8bc264ac0d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g namespace: kserve-ci-e2e-test resourceVersion: "21553" uid: e3a6010f-eee8-4a8c-894e-d5b21bec7a31 kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:11:06Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g.18c7687dba2d6938 namespace: kserve-ci-e2e-test resourceVersion: "21585" uid: 80a3fa2f-7663-49ff-aa40-832f2277b457 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: e3a6010f-eee8-4a8c-894e-d5b21bec7a31 kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:06Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g.18c7687dbc2d6a9e namespace: kserve-ci-e2e-test resourceVersion: "21590" uid: e37ae078-3d22-4aa6-8a2b-245955500b55 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: e3a6010f-eee8-4a8c-894e-d5b21bec7a31 kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:06Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g.18c7687dbee41102 namespace: kserve-ci-e2e-test resourceVersion: "21592" uid: a7052ce6-18b3-44cd-8688-e3af5ecbd4f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: e3a6010f-eee8-4a8c-894e-d5b21bec7a31 kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:06Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g.18c7687dbf5ee9d3 namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: 79317416-1103-4833-bb73-95e9c9057727 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: e3a6010f-eee8-4a8c-894e-d5b21bec7a31 kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:06Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g.18c7687dbf69c3b1 namespace: kserve-ci-e2e-test resourceVersion: "21595" uid: 61c5f0bb-ea77-48e9-9631-80fa4aa429c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: e3a6010f-eee8-4a8c-894e-d5b21bec7a31 kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:06Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g.18c7687dc2a820d8 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 769f4bcb-ad62-4d16-818f-0029ba270999 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: e3a6010f-eee8-4a8c-894e-d5b21bec7a31 kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:06Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g.18c7687dc323fa61 namespace: kserve-ci-e2e-test resourceVersion: "21597" uid: d421ca63-1863-4606-a964-3f0d71809a5d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: e3a6010f-eee8-4a8c-894e-d5b21bec7a31 kind: Event lastTimestamp: "2026-07-31T15:20:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:22Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g.18c7687dc37e5f9e namespace: kserve-ci-e2e-test resourceVersion: "26286" uid: 4282272f-8b3b-4f92-b057-22607844296b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: e3a6010f-eee8-4a8c-894e-d5b21bec7a31 kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:20Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g.18c768feb007989a namespace: kserve-ci-e2e-test resourceVersion: "26233" uid: 33dc1533-e31f-44b7-816f-ca2ecbd509fd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: e3a6010f-eee8-4a8c-894e-d5b21bec7a31 kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:20Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g.18c768feb0081ee1 namespace: kserve-ci-e2e-test resourceVersion: "26234" uid: dffc96e7-6a62-4639-83cd-48098330ce37 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: e3a6010f-eee8-4a8c-894e-d5b21bec7a31 kind: Event lastTimestamp: "2026-07-31T15:20:22Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:22Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g.18c768ff37e0eea3 namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 156ce69c-a1c2-4c51-8768-e7981d4c8890 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-06a6c-predictor-5766f5fbbc namespace: kserve-ci-e2e-test resourceVersion: "21535" uid: 13180e59-b076-42d5-8967-911e24c0bd7f kind: Event lastTimestamp: "2026-07-31T15:11:05Z" message: 'Created pod: error-404-isvc-06a6c-predictor-5766f5fbbc-h8s7g' metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:11:05Z" name: error-404-isvc-06a6c-predictor-5766f5fbbc.18c7687d7b9a9f75 namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: 2cc5098b-cbb1-47b1-a8a7-4c669f5910a1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-06a6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21532" uid: 9eed5faf-90f4-4dd2-8ae4-be35a5d7bbbd kind: Event lastTimestamp: "2026-07-31T15:11:05Z" message: Scaled up replica set error-404-isvc-06a6c-predictor-5766f5fbbc from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:11:05Z" name: error-404-isvc-06a6c-predictor.18c7687d79f5cf9d namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: 0935f7f3-287d-4c2e-b8ab-086697c4133d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:11:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-06a6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: dc9486a5-4296-4f22-b37c-fa08e796ca04 kind: Event lastTimestamp: "2026-07-31T15:11:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:11:50Z" name: error-404-isvc-06a6c-predictor.18c76880f86f9531 namespace: kserve-ci-e2e-test resourceVersion: "21978" uid: d1e69a53-3864-4836-b5cf-a544567a371b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:11:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-06a6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: dc9486a5-4296-4f22-b37c-fa08e796ca04 kind: Event lastTimestamp: "2026-07-31T15:11:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:11:50Z" name: error-404-isvc-06a6c-predictor.18c76880f8dd9a88 namespace: kserve-ci-e2e-test resourceVersion: "21979" uid: ba6e41b2-f043-4953-a831-bf191c91059f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-06a6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21726" uid: dc9486a5-4296-4f22-b37c-fa08e796ca04 kind: Event lastTimestamp: "2026-07-31T15:12:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:20Z" name: error-404-isvc-06a6c-predictor.18c7688b74e545ec namespace: kserve-ci-e2e-test resourceVersion: "22456" uid: 95083c1f-8b31-4300-ae7e-d5fc568cd563 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-06a6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21726" uid: dc9486a5-4296-4f22-b37c-fa08e796ca04 kind: Event lastTimestamp: "2026-07-31T15:12:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:20Z" name: error-404-isvc-06a6c-predictor.18c7688b7539e5a5 namespace: kserve-ci-e2e-test resourceVersion: "22457" uid: fe39ec79-d8dc-496b-b3b5-c299681ef6d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-06a6c namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: 1c7d2b94-5538-4f30-9442-2e360a3ed24f kind: Event lastTimestamp: "2026-07-31T15:12:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:12:02Z" name: error-404-isvc-06a6c.18c7687d79febb8d namespace: kserve-ci-e2e-test resourceVersion: "22130" uid: f2b2b901-aef7-456d-a103-d705fadfaa5f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-06a6c namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: 1c7d2b94-5538-4f30-9442-2e360a3ed24f kind: Event lastTimestamp: "2026-07-31T15:11:05Z" message: 'Failed to update status for InferenceService "error-404-isvc-06a6c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-06a6c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:11:05Z" name: error-404-isvc-06a6c.18c7687d7a7aba47 namespace: kserve-ci-e2e-test resourceVersion: "21540" uid: 60c9c97a-e397-4572-ae58-fe35271a1a8b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-06a6c namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: 1c7d2b94-5538-4f30-9442-2e360a3ed24f kind: Event lastTimestamp: "2026-07-31T15:11:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-06a6c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:11:05Z" name: error-404-isvc-06a6c.18c7687d7a7b4346 namespace: kserve-ci-e2e-test resourceVersion: "21541" uid: 41fd2922-6ba6-46e4-b447-28844e48a15c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-06a6c namespace: kserve-ci-e2e-test resourceVersion: "22128" uid: 1c7d2b94-5538-4f30-9442-2e360a3ed24f kind: Event lastTimestamp: "2026-07-31T15:12:02Z" message: InferenceService [error-404-isvc-06a6c] is Ready metadata: creationTimestamp: "2026-07-31T15:12:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:12:02Z" name: error-404-isvc-06a6c.18c7688ad610f468 namespace: kserve-ci-e2e-test resourceVersion: "22129" uid: d8b71225-f747-4a85-a683-115bda5ffed3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:09:55.055649Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-49957-predictor-bc66879c5-mgq5g namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 251c1551-b708-4f55-a230-5578c7409efc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-49957-predictor-bc66879c5-mgq5g to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-49957-predictor-bc66879c5-mgq5g.18c7686d18fd2e76 namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: 4e41f989-2e3c-4c85-b35d-3a3798c0be29 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-49957-predictor-bc66879c5-mgq5g namespace: kserve-ci-e2e-test resourceVersion: "20655" uid: 251c1551-b708-4f55-a230-5578c7409efc kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-49957-predictor-bc66879c5-mgq5g.18c7686d3333f0af namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: ba841536-42c9-4d3e-984a-44432f70c8ff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49957-predictor-bc66879c5-mgq5g namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 251c1551-b708-4f55-a230-5578c7409efc kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-49957-predictor-bc66879c5-mgq5g.18c7686d34aa412a namespace: kserve-ci-e2e-test resourceVersion: "20686" uid: a7f35b82-94c2-4748-8e05-dffed4bbea3a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49957-predictor-bc66879c5-mgq5g namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 251c1551-b708-4f55-a230-5578c7409efc kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-49957-predictor-bc66879c5-mgq5g.18c7686d37d666e2 namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: b2b05603-d7f0-486e-955a-5eb787dbd6f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49957-predictor-bc66879c5-mgq5g namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 251c1551-b708-4f55-a230-5578c7409efc kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-49957-predictor-bc66879c5-mgq5g.18c7686d3864b3aa namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: d7d71011-6e69-44ee-aef1-acfcf3f6f407 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49957-predictor-bc66879c5-mgq5g namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 251c1551-b708-4f55-a230-5578c7409efc kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-49957-predictor-bc66879c5-mgq5g.18c7686d386e80f7 namespace: kserve-ci-e2e-test resourceVersion: "20693" uid: a667ee72-8a94-4d8c-a0b2-730b74091249 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49957-predictor-bc66879c5-mgq5g namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 251c1551-b708-4f55-a230-5578c7409efc kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-49957-predictor-bc66879c5-mgq5g.18c7686d3b2f08bb namespace: kserve-ci-e2e-test resourceVersion: "20694" uid: 985aafa6-1afe-4ff5-b12d-362e11c226fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49957-predictor-bc66879c5-mgq5g namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 251c1551-b708-4f55-a230-5578c7409efc kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-49957-predictor-bc66879c5-mgq5g.18c7686d3ba93970 namespace: kserve-ci-e2e-test resourceVersion: "20695" uid: 4d5a942f-4717-4227-9cf1-df966013339c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:09:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49957-predictor-bc66879c5-mgq5g namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 251c1551-b708-4f55-a230-5578c7409efc kind: Event lastTimestamp: "2026-07-31T15:10:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:09:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:32Z" name: error-404-isvc-49957-predictor-bc66879c5-mgq5g.18c7686d64a74ab5 namespace: kserve-ci-e2e-test resourceVersion: "20995" uid: 7b0563e8-2e65-4ca6-b4c5-3ca0d35cb32f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49957-predictor-bc66879c5-mgq5g namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 251c1551-b708-4f55-a230-5578c7409efc kind: Event lastTimestamp: "2026-07-31T15:11:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:05Z" name: error-404-isvc-49957-predictor-bc66879c5-mgq5g.18c7687d767e7319 namespace: kserve-ci-e2e-test resourceVersion: "21521" uid: 88bbccc4-3ac1-45ac-97f1-b602d75b9814 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49957-predictor-bc66879c5-mgq5g namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 251c1551-b708-4f55-a230-5578c7409efc kind: Event lastTimestamp: "2026-07-31T15:11:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:05Z" name: error-404-isvc-49957-predictor-bc66879c5-mgq5g.18c7687d767f8169 namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: cb35fa14-4be7-4e32-a19a-a71213872b3a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49957-predictor-bc66879c5-mgq5g namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: 251c1551-b708-4f55-a230-5578c7409efc kind: Event lastTimestamp: "2026-07-31T15:11:07Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:07Z" name: error-404-isvc-49957-predictor-bc66879c5-mgq5g.18c7687decce021d namespace: kserve-ci-e2e-test resourceVersion: "21611" uid: 39cc737f-d90c-4f9e-9d6b-25b2f4d131e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-49957-predictor-bc66879c5 namespace: kserve-ci-e2e-test resourceVersion: "20634" uid: 84595631-2df0-4946-a66b-010d836d8c45 kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: 'Created pod: error-404-isvc-49957-predictor-bc66879c5-mgq5g' metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-49957-predictor-bc66879c5.18c7686d189427f8 namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 72fdfb01-46e5-413e-aa11-c9345604a599 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-49957-predictor namespace: kserve-ci-e2e-test resourceVersion: "20631" uid: 150756da-0e36-447b-8c09-4f44f21f3de9 kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: Scaled up replica set error-404-isvc-49957-predictor-bc66879c5 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-49957-predictor.18c7686d16de15b7 namespace: kserve-ci-e2e-test resourceVersion: "20638" uid: c30564e7-9706-4f25-931b-13c2ce1ac298 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:10:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-49957-predictor namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: ba674f9b-5f84-4b29-bba4-c2949b31d997 kind: Event lastTimestamp: "2026-07-31T15:10:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:40Z" name: error-404-isvc-49957-predictor.18c76870953d2365 namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: 94355c41-6e02-45ea-94f6-5abc1a920dac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:10:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-49957-predictor namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: ba674f9b-5f84-4b29-bba4-c2949b31d997 kind: Event lastTimestamp: "2026-07-31T15:10:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:10:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:40Z" name: error-404-isvc-49957-predictor.18c7687095a70027 namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: 7c7ac086-b06f-4af3-aa1a-913f958f3b1e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-49957-predictor namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: ba674f9b-5f84-4b29-bba4-c2949b31d997 kind: Event lastTimestamp: "2026-07-31T15:10:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:55Z" name: error-404-isvc-49957-predictor.18c7687b12184c23 namespace: kserve-ci-e2e-test resourceVersion: "21348" uid: 01ba8ea5-3c27-4fdc-867f-39259cfe3fbd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-49957-predictor namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: ba674f9b-5f84-4b29-bba4-c2949b31d997 kind: Event lastTimestamp: "2026-07-31T15:10:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:55Z" name: error-404-isvc-49957-predictor.18c7687b1295b723 namespace: kserve-ci-e2e-test resourceVersion: "21350" uid: 41a1641b-cf48-4d5c-bada-99eb9b1ae839 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-49957 namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 00a391a8-5e05-4486-a4dd-85f023a6813e kind: Event lastTimestamp: "2026-07-31T15:10:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:10:42Z" name: error-404-isvc-49957.18c7686d16dff042 namespace: kserve-ci-e2e-test resourceVersion: "21092" uid: 18b97def-cc42-4b7e-a60d-1d5c36c28643 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-49957 namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 00a391a8-5e05-4486-a4dd-85f023a6813e kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: 'Failed to update status for InferenceService "error-404-isvc-49957": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-49957": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-49957.18c7686d17796c47 namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: 40d67682-e66d-4bc1-9068-518abb3acb20 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-49957 namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 00a391a8-5e05-4486-a4dd-85f023a6813e kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-49957": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-49957.18c7686d1779af60 namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: ebb8251b-7947-44f5-8e4b-7b03e89cc580 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-49957 namespace: kserve-ci-e2e-test resourceVersion: "21089" uid: 00a391a8-5e05-4486-a4dd-85f023a6813e kind: Event lastTimestamp: "2026-07-31T15:10:42Z" message: InferenceService [error-404-isvc-49957] is Ready metadata: creationTimestamp: "2026-07-31T15:10:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:10:42Z" name: error-404-isvc-49957.18c7687822c495c2 namespace: kserve-ci-e2e-test resourceVersion: "21090" uid: 4cd2c1ee-fc8e-4a49-956a-2162426489e1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:21:30.492700Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv namespace: kserve-ci-e2e-test resourceVersion: "26962" uid: 1c74c8bc-28a4-467a-a807-0e14cfdad979 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:21:30Z" name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv.18c7690f04445c7e namespace: kserve-ci-e2e-test resourceVersion: "26967" uid: 6c35c14c-d398-44f6-8378-e95f25725e90 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv namespace: kserve-ci-e2e-test resourceVersion: "26977" uid: 1c74c8bc-28a4-467a-a807-0e14cfdad979 kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:21:30Z" name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv.18c7690f1f3c9931 namespace: kserve-ci-e2e-test resourceVersion: "27040" uid: 02823c3f-e1f6-4fdf-bb1c-61bbced61448 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: 1c74c8bc-28a4-467a-a807-0e14cfdad979 kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:30Z" name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv.18c7690f20fd3ed6 namespace: kserve-ci-e2e-test resourceVersion: "27047" uid: 5626340f-296f-44e7-ac5c-f6989b3ba5dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: 1c74c8bc-28a4-467a-a807-0e14cfdad979 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv.18c7690f248b38ad namespace: kserve-ci-e2e-test resourceVersion: "27077" uid: 8cf54b15-7acc-4785-b23c-fb17729e400c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: 1c74c8bc-28a4-467a-a807-0e14cfdad979 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv.18c7690f2520116e namespace: kserve-ci-e2e-test resourceVersion: "27079" uid: 9780a5b3-96a8-4cc8-b0d1-9dadbdd0bd50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: 1c74c8bc-28a4-467a-a807-0e14cfdad979 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv.18c7690f252a5ed2 namespace: kserve-ci-e2e-test resourceVersion: "27081" uid: f4477e8a-41ff-4ca9-93f1-3e6a76a6cf34 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: 1c74c8bc-28a4-467a-a807-0e14cfdad979 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv.18c7690f2cc91d7f namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: 0ed4aab8-e6a2-47dd-892a-251cc3365f27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: 1c74c8bc-28a4-467a-a807-0e14cfdad979 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv.18c7690f2d5928d3 namespace: kserve-ci-e2e-test resourceVersion: "27096" uid: 1fb359e5-1e0f-4893-bfe5-fcd2a8397bf8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: 1c74c8bc-28a4-467a-a807-0e14cfdad979 kind: Event lastTimestamp: "2026-07-31T15:22:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:22:19Z" name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv.18c7690f5e6f20b3 namespace: kserve-ci-e2e-test resourceVersion: "27560" uid: 1e8bd015-3571-4f6b-819e-5db8ae128fe0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: 1c74c8bc-28a4-467a-a807-0e14cfdad979 kind: Event lastTimestamp: "2026-07-31T15:38:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:38:49Z" name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv.18c76a0102fb66e3 namespace: kserve-ci-e2e-test resourceVersion: "34563" uid: 8cd585fc-e524-4390-8285-647c9d29e6cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: 1c74c8bc-28a4-467a-a807-0e14cfdad979 kind: Event lastTimestamp: "2026-07-31T15:38:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:38:49Z" name: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv.18c76a0102fc30ce namespace: kserve-ci-e2e-test resourceVersion: "34564" uid: 62657af6-c17a-4026-8612-4a32d1510062 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7902a-predictor-c5f95cb9c namespace: kserve-ci-e2e-test resourceVersion: "26953" uid: 57f681d8-2f6d-4c9d-9e40-aef21abae11c kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: 'Created pod: error-404-isvc-7902a-predictor-c5f95cb9c-sqvtv' metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:30Z" name: error-404-isvc-7902a-predictor-c5f95cb9c.18c7690f03bdeeff namespace: kserve-ci-e2e-test resourceVersion: "26964" uid: b35f5e64-db32-4270-8ea1-4e548c6eef9f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7902a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26950" uid: 8c19ec6b-5578-47be-8801-0308c60b1543 kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Scaled up replica set error-404-isvc-7902a-predictor-c5f95cb9c from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:30Z" name: error-404-isvc-7902a-predictor.18c7690f013557f9 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 9ced1598-537d-41f4-a4ab-0c75b57d5ef5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:21:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7902a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 09bfed80-9dd4-491e-933f-13a3647975a2 kind: Event lastTimestamp: "2026-07-31T15:22:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:21:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:00Z" name: error-404-isvc-7902a-predictor.18c769127faaad1c namespace: kserve-ci-e2e-test resourceVersion: "27400" uid: 62fa2d43-81a8-47b6-81e5-555598fcfb9f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:21:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7902a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 09bfed80-9dd4-491e-933f-13a3647975a2 kind: Event lastTimestamp: "2026-07-31T15:22:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:21:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:00Z" name: error-404-isvc-7902a-predictor.18c7691280062b0b namespace: kserve-ci-e2e-test resourceVersion: "27401" uid: c2af549c-b2bc-4c9f-bae8-153dc7a4958a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T15:22:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7902a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27268" uid: 09bfed80-9dd4-491e-933f-13a3647975a2 kind: Event lastTimestamp: "2026-07-31T15:23:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:23:00Z" name: error-404-isvc-7902a-predictor.18c769197e4f44a4 namespace: kserve-ci-e2e-test resourceVersion: "27975" uid: 6d0ebd20-8838-4ec8-81de-08039460ec63 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T15:22:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7902a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27268" uid: 09bfed80-9dd4-491e-933f-13a3647975a2 kind: Event lastTimestamp: "2026-07-31T15:23:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:23:00Z" name: error-404-isvc-7902a-predictor.18c769197ea85d8c namespace: kserve-ci-e2e-test resourceVersion: "27977" uid: 002ec1f8-5bb8-4641-a10f-fab1f3e82ce6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7902a namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 3f22942d-8f40-4116-ab32-c329b886c79a kind: Event lastTimestamp: "2026-07-31T15:22:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:22:29Z" name: error-404-isvc-7902a.18c7690f0165f654 namespace: kserve-ci-e2e-test resourceVersion: "27653" uid: e4d5bde7-ecbd-43a0-9653-426a69a53eeb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7902a namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 3f22942d-8f40-4116-ab32-c329b886c79a kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: 'Failed to update status for InferenceService "error-404-isvc-7902a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7902a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:21:30Z" name: error-404-isvc-7902a.18c7690f01e29b33 namespace: kserve-ci-e2e-test resourceVersion: "26958" uid: f242220a-eeb6-4546-ab62-f6980bb2088e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7902a namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 3f22942d-8f40-4116-ab32-c329b886c79a kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7902a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:21:30Z" name: error-404-isvc-7902a.18c7690f01e2cf7f namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 1c50925e-47ba-4856-9eac-46836e190d13 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7902a namespace: kserve-ci-e2e-test resourceVersion: "27650" uid: 3f22942d-8f40-4116-ab32-c329b886c79a kind: Event lastTimestamp: "2026-07-31T15:22:29Z" message: InferenceService [error-404-isvc-7902a] is Ready metadata: creationTimestamp: "2026-07-31T15:22:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:22:29Z" name: error-404-isvc-7902a.18c7691caa738475 namespace: kserve-ci-e2e-test resourceVersion: "27651" uid: 2e80ffb4-af9d-45e5-bbf5-dce6cd648370 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:00:40.304869Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx namespace: kserve-ci-e2e-test resourceVersion: "16251" uid: f7735513-ebb7-4d09-baa6-7b74d68c942f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx.18c767ebef440855 namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: d8ce59af-a70a-47a2-a6bb-1e7c2fddbffb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: f7735513-ebb7-4d09-baa6-7b74d68c942f kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx.18c767ec092344a7 namespace: kserve-ci-e2e-test resourceVersion: "16293" uid: 431c5386-ed47-426b-a9b7-914911cec204 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: f7735513-ebb7-4d09-baa6-7b74d68c942f kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx.18c767ec0a9b1676 namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 0583568f-a10c-4830-9669-cb2641e681cb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: f7735513-ebb7-4d09-baa6-7b74d68c942f kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx.18c767ec0d31e21f namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 2ee7db10-3f4e-4660-904a-2c8af62ea7a7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: f7735513-ebb7-4d09-baa6-7b74d68c942f kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx.18c767ec0daaa4b1 namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: 1b21c8e2-eb74-477f-8294-bbe932a82337 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: f7735513-ebb7-4d09-baa6-7b74d68c942f kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx.18c767ec0db56192 namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 6c8eb8e2-cb0f-41c3-9703-21953a247b31 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: f7735513-ebb7-4d09-baa6-7b74d68c942f kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx.18c767ec113e4354 namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: 5090b879-da26-4648-8379-27c8e97c1df9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: f7735513-ebb7-4d09-baa6-7b74d68c942f kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx.18c767ec11b948a7 namespace: kserve-ci-e2e-test resourceVersion: "16307" uid: ce3cd231-5d42-4226-9974-832d59307427 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T15:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: f7735513-ebb7-4d09-baa6-7b74d68c942f kind: Event lastTimestamp: "2026-07-31T15:09:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:57Z" name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx.18c767ec276ca954 namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: cea91794-3ee9-4e16-8ddd-b4380ccba5e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: f7735513-ebb7-4d09-baa6-7b74d68c942f kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx.18c7686d1c9c5166 namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 6fdfc322-4469-4c14-9e7d-5a9e2f7c671f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: f7735513-ebb7-4d09-baa6-7b74d68c942f kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx.18c7686d1c9cb138 namespace: kserve-ci-e2e-test resourceVersion: "20672" uid: 97598c32-90a9-4720-b475-ea64bae12ff3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: f7735513-ebb7-4d09-baa6-7b74d68c942f kind: Event lastTimestamp: "2026-07-31T15:09:57Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:09:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:57Z" name: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx.18c7686d9c1b722d namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: 172f1df6-f75d-4b2a-98bb-048ce9611cc8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7b0b5-predictor-fc66d6ff namespace: kserve-ci-e2e-test resourceVersion: "16238" uid: a3ed54aa-02c3-4be3-ba2a-dcd6d0e8417e kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: 'Created pod: error-404-isvc-7b0b5-predictor-fc66d6ff-kgclx' metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-7b0b5-predictor-fc66d6ff.18c767ebeed52b0f namespace: kserve-ci-e2e-test resourceVersion: "16255" uid: 35a5f900-efda-4b94-9eb0-12d8c99c7717 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7b0b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: 724d6058-5496-4cc3-97a6-31edacd46013 kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Scaled up replica set error-404-isvc-7b0b5-predictor-fc66d6ff from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-7b0b5-predictor.18c767ebec1322c9 namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: 0a8442da-8cd0-4de7-832b-da16e2c97e9f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:00:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b0b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: e8abe85e-1b0a-4503-b2d4-78b731b70690 kind: Event lastTimestamp: "2026-07-31T15:01:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:00:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:10Z" name: error-404-isvc-7b0b5-predictor.18c767ef6aae5a8b namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: d6865039-d7c8-4386-9ef6-41da695639cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:00:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b0b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: e8abe85e-1b0a-4503-b2d4-78b731b70690 kind: Event lastTimestamp: "2026-07-31T15:01:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:00:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:10Z" name: error-404-isvc-7b0b5-predictor.18c767ef6b2ad02c namespace: kserve-ci-e2e-test resourceVersion: "16591" uid: 9af54b91-55fb-4fc6-a815-84477e1078f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:01:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b0b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: e8abe85e-1b0a-4503-b2d4-78b731b70690 kind: Event lastTimestamp: "2026-07-31T15:01:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:40Z" name: error-404-isvc-7b0b5-predictor.18c767f668e440a9 namespace: kserve-ci-e2e-test resourceVersion: "17072" uid: d66b1629-ad4f-4628-b487-c364e73363c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:01:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b0b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: e8abe85e-1b0a-4503-b2d4-78b731b70690 kind: Event lastTimestamp: "2026-07-31T15:01:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:40Z" name: error-404-isvc-7b0b5-predictor.18c767f669b82176 namespace: kserve-ci-e2e-test resourceVersion: "17073" uid: 58658479-7702-4f39-9336-6d11a627b4f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b0b5 namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 9e7f47ca-d90e-4d07-ab20-d2c29852d23a kind: Event lastTimestamp: "2026-07-31T15:01:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:01:27Z" name: error-404-isvc-7b0b5.18c767ebec4e19d8 namespace: kserve-ci-e2e-test resourceVersion: "16772" uid: e8d0972a-8678-4d78-a97b-525ca942a1a0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b0b5 namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 9e7f47ca-d90e-4d07-ab20-d2c29852d23a kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: 'Failed to update status for InferenceService "error-404-isvc-7b0b5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7b0b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-7b0b5.18c767ebecb71a27 namespace: kserve-ci-e2e-test resourceVersion: "16246" uid: 74df7779-a63f-4e4e-8f0f-5e469013d313 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b0b5 namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 9e7f47ca-d90e-4d07-ab20-d2c29852d23a kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7b0b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-7b0b5.18c767ebecb76472 namespace: kserve-ci-e2e-test resourceVersion: "16249" uid: 6490b623-c11d-4c8e-a3e4-cfd4f5ebce9d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b0b5 namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: 9e7f47ca-d90e-4d07-ab20-d2c29852d23a kind: Event lastTimestamp: "2026-07-31T15:01:27Z" message: InferenceService [error-404-isvc-7b0b5] is Ready metadata: creationTimestamp: "2026-07-31T15:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:01:27Z" name: error-404-isvc-7b0b5.18c767f6e353309a namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: 84b2b214-8e36-4041-8e1b-2fff6deeef12 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:12:15.423945Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7b982-predictor-d56595d99-q8npg namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: 420b74ef-0f55-4f6f-a974-beff045533bb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7b982-predictor-d56595d99-q8npg to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-7b982-predictor-d56595d99-q8npg.18c7688dc79764ae namespace: kserve-ci-e2e-test resourceVersion: "22364" uid: 6d61ffe8-0992-4b84-a25b-1749f73bb6f2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7b982-predictor-d56595d99-q8npg namespace: kserve-ci-e2e-test resourceVersion: "22369" uid: 420b74ef-0f55-4f6f-a974-beff045533bb kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-7b982-predictor-d56595d99-q8npg.18c7688de16ac65b namespace: kserve-ci-e2e-test resourceVersion: "22383" uid: 80befa3f-670b-49de-b7b2-2b54f2b62ab7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b982-predictor-d56595d99-q8npg namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 420b74ef-0f55-4f6f-a974-beff045533bb kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-7b982-predictor-d56595d99-q8npg.18c7688de2db598f namespace: kserve-ci-e2e-test resourceVersion: "22385" uid: b5e371fb-b348-479e-b66e-8fac34780713 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b982-predictor-d56595d99-q8npg namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 420b74ef-0f55-4f6f-a974-beff045533bb kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-7b982-predictor-d56595d99-q8npg.18c7688de5ba83d2 namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: 2fdf8292-756d-4bf0-b8a9-e9550fceae51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b982-predictor-d56595d99-q8npg namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 420b74ef-0f55-4f6f-a974-beff045533bb kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-7b982-predictor-d56595d99-q8npg.18c7688de6327299 namespace: kserve-ci-e2e-test resourceVersion: "22388" uid: 4acfe391-1cbd-4d0f-a671-95b6b2bc0a41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b982-predictor-d56595d99-q8npg namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 420b74ef-0f55-4f6f-a974-beff045533bb kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-7b982-predictor-d56595d99-q8npg.18c7688de63c6c32 namespace: kserve-ci-e2e-test resourceVersion: "22389" uid: 453df86e-2477-4eb6-b327-8495920578f6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b982-predictor-d56595d99-q8npg namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 420b74ef-0f55-4f6f-a974-beff045533bb kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-7b982-predictor-d56595d99-q8npg.18c7688de8f97391 namespace: kserve-ci-e2e-test resourceVersion: "22390" uid: 27a555db-7a87-4fe8-a8fc-f60c31141c54 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b982-predictor-d56595d99-q8npg namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 420b74ef-0f55-4f6f-a974-beff045533bb kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-7b982-predictor-d56595d99-q8npg.18c7688de97f8fcc namespace: kserve-ci-e2e-test resourceVersion: "22391" uid: eb409122-2d10-4913-989c-da78c74f2916 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T15:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b982-predictor-d56595d99-q8npg namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 420b74ef-0f55-4f6f-a974-beff045533bb kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: error-404-isvc-7b982-predictor-d56595d99-q8npg.18c7688e20cd6ba2 namespace: kserve-ci-e2e-test resourceVersion: "27140" uid: 08f1f643-1ed0-4dad-83a6-1a4e33621bdf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b982-predictor-d56595d99-q8npg namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 420b74ef-0f55-4f6f-a974-beff045533bb kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:30Z" name: error-404-isvc-7b982-predictor-d56595d99-q8npg.18c7690efa5b993b namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: 5551bd25-b28c-4976-a6a4-a8ad98f6d0ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b982-predictor-d56595d99-q8npg namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 420b74ef-0f55-4f6f-a974-beff045533bb kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:30Z" name: error-404-isvc-7b982-predictor-d56595d99-q8npg.18c7690efa5c3edb namespace: kserve-ci-e2e-test resourceVersion: "26921" uid: 49cbd1f7-11f6-46e7-ae9b-3e1ba18bd55a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b982-predictor-d56595d99-q8npg namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 420b74ef-0f55-4f6f-a974-beff045533bb kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: error-404-isvc-7b982-predictor-d56595d99-q8npg.18c7690f954efe57 namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: 4f4ae0be-50a5-4a1e-9593-9dfc4001a3b6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7b982-predictor-d56595d99 namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: a63f175c-246f-4fca-92c3-9c230d11beb4 kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: 'Created pod: error-404-isvc-7b982-predictor-d56595d99-q8npg' metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-7b982-predictor-d56595d99.18c7688dc73bbb34 namespace: kserve-ci-e2e-test resourceVersion: "22362" uid: 503f4864-d93b-4722-8c7b-81458af7591a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7b982-predictor namespace: kserve-ci-e2e-test resourceVersion: "22348" uid: 081d362b-a4bf-41ee-9119-632a11596cef kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: Scaled up replica set error-404-isvc-7b982-predictor-d56595d99 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-7b982-predictor.18c7688dc4d2fa94 namespace: kserve-ci-e2e-test resourceVersion: "22352" uid: 39f9aa4d-67d3-4e61-8e91-95a2e80fa30c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:12:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b982-predictor namespace: kserve-ci-e2e-test resourceVersion: "22350" uid: 7e89eaac-adc5-4b36-9fd8-980d29afcc6a kind: Event lastTimestamp: "2026-07-31T15:12:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:45Z" name: error-404-isvc-7b982-predictor.18c76891435f10cc namespace: kserve-ci-e2e-test resourceVersion: "22654" uid: cb8be20b-4fd1-429c-bcac-02f22b09244e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:12:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b982-predictor namespace: kserve-ci-e2e-test resourceVersion: "22350" uid: 7e89eaac-adc5-4b36-9fd8-980d29afcc6a kind: Event lastTimestamp: "2026-07-31T15:12:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:45Z" name: error-404-isvc-7b982-predictor.18c7689143c62578 namespace: kserve-ci-e2e-test resourceVersion: "22655" uid: efa9bf24-9744-4385-a3b5-f3d1fa233e7c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:13:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b982-predictor namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 7e89eaac-adc5-4b36-9fd8-980d29afcc6a kind: Event lastTimestamp: "2026-07-31T15:13:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:13:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:13:30Z" name: error-404-isvc-7b982-predictor.18c76898413f0a88 namespace: kserve-ci-e2e-test resourceVersion: "23043" uid: 66340dd6-6bd1-4b0b-ad9f-93368107b7f6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:13:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b982-predictor namespace: kserve-ci-e2e-test resourceVersion: "22539" uid: 7e89eaac-adc5-4b36-9fd8-980d29afcc6a kind: Event lastTimestamp: "2026-07-31T15:13:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:13:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:13:30Z" name: error-404-isvc-7b982-predictor.18c7689841b08838 namespace: kserve-ci-e2e-test resourceVersion: "23044" uid: 9c00e1e9-c877-49a2-a0e2-ed27ff6f8a7a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b982 namespace: kserve-ci-e2e-test resourceVersion: "22331" uid: 52f951cd-a5f9-43a3-9088-d7aabaf3ff9e kind: Event lastTimestamp: "2026-07-31T15:13:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:13:03Z" name: error-404-isvc-7b982.18c7688dc4d0f7c5 namespace: kserve-ci-e2e-test resourceVersion: "22805" uid: 1d013221-20dd-46ff-abe7-bec0d2ac6d3f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b982 namespace: kserve-ci-e2e-test resourceVersion: "22331" uid: 52f951cd-a5f9-43a3-9088-d7aabaf3ff9e kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-7b982": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7b982": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-7b982.18c7688dc55a6987 namespace: kserve-ci-e2e-test resourceVersion: "22356" uid: 748ecfd4-f8ed-4a9a-a3f1-ad3e57ce8415 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b982 namespace: kserve-ci-e2e-test resourceVersion: "22331" uid: 52f951cd-a5f9-43a3-9088-d7aabaf3ff9e kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7b982": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-7b982.18c7688dc55ab627 namespace: kserve-ci-e2e-test resourceVersion: "22357" uid: 4a12319c-849e-444e-b2b4-b3903ce4eaff reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b982 namespace: kserve-ci-e2e-test resourceVersion: "22799" uid: 52f951cd-a5f9-43a3-9088-d7aabaf3ff9e kind: Event lastTimestamp: "2026-07-31T15:13:03Z" message: InferenceService [error-404-isvc-7b982] is Ready metadata: creationTimestamp: "2026-07-31T15:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:13:03Z" name: error-404-isvc-7b982.18c76898dc8ee018 namespace: kserve-ci-e2e-test resourceVersion: "22800" uid: c548a2a9-5f4d-480b-a824-97fca5228c6e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:10:51.044033Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: 61ffb3d0-d0f8-4504-b0a0-b5d16b80a0c0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a1442-predictor-84bf5c887f-bmcrh to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:10:51Z" name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh.18c7687a2228037e namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 06d36e96-26a5-49e2-abd8-6f2c6fe148e0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: 61ffb3d0-d0f8-4504-b0a0-b5d16b80a0c0 kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:10:51Z" name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh.18c7687a3bd91dc3 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 8a5f96b8-7eb3-4d13-b524-359c4b53a659 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 61ffb3d0-d0f8-4504-b0a0-b5d16b80a0c0 kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:51Z" name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh.18c7687a3d47cb4b namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: 8ea29d02-6039-440c-994b-108bc8f5d2c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 61ffb3d0-d0f8-4504-b0a0-b5d16b80a0c0 kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:51Z" name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh.18c7687a413b2a8e namespace: kserve-ci-e2e-test resourceVersion: "21271" uid: 1e296334-6c42-4e00-8d72-0a96c01f902b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 61ffb3d0-d0f8-4504-b0a0-b5d16b80a0c0 kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:51Z" name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh.18c7687a41aecdc0 namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: 54153695-e175-4a67-a014-b8d593e061c8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 61ffb3d0-d0f8-4504-b0a0-b5d16b80a0c0 kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:51Z" name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh.18c7687a41b80e11 namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 3d287d35-9d0e-451b-bdec-4d389b517e07 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 61ffb3d0-d0f8-4504-b0a0-b5d16b80a0c0 kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:51Z" name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh.18c7687a4451e8ce namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 82e4db47-d810-4a3e-afe6-f34dc6623e1f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 61ffb3d0-d0f8-4504-b0a0-b5d16b80a0c0 kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:51Z" name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh.18c7687a44c76080 namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: facc032a-af2a-4064-a9ed-ea9a145030a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-31T15:10:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 61ffb3d0-d0f8-4504-b0a0-b5d16b80a0c0 kind: Event lastTimestamp: "2026-07-31T15:12:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:18Z" name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh.18c7687a7c9d6122 namespace: kserve-ci-e2e-test resourceVersion: "22432" uid: 8fa91341-7373-455d-aed1-1c06c8ecc23e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 61ffb3d0-d0f8-4504-b0a0-b5d16b80a0c0 kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh.18c7688dc174d839 namespace: kserve-ci-e2e-test resourceVersion: "22337" uid: 4bdbbcfc-faff-4d18-9199-0952a9c70199 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 61ffb3d0-d0f8-4504-b0a0-b5d16b80a0c0 kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:15Z" name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh.18c7688dc1755792 namespace: kserve-ci-e2e-test resourceVersion: "22340" uid: 91551e2b-fe3c-43a8-b240-9f95018b1b63 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 61ffb3d0-d0f8-4504-b0a0-b5d16b80a0c0 kind: Event lastTimestamp: "2026-07-31T15:12:18Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:12:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:18Z" name: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh.18c7688e82eac7fc namespace: kserve-ci-e2e-test resourceVersion: "22430" uid: 6e29911c-b927-41ec-a8f4-f41d75e9a58d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a1442-predictor-84bf5c887f namespace: kserve-ci-e2e-test resourceVersion: "21234" uid: ee2de20e-7cfd-4493-906c-358a5415cc04 kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: 'Created pod: error-404-isvc-a1442-predictor-84bf5c887f-bmcrh' metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:51Z" name: error-404-isvc-a1442-predictor-84bf5c887f.18c7687a2182037a namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 246863c6-3f92-4628-9f07-15928e5e9ac6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a1442-predictor namespace: kserve-ci-e2e-test resourceVersion: "21231" uid: f62aaf63-91b5-4a9e-ac16-a037ef9f0ed0 kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: Scaled up replica set error-404-isvc-a1442-predictor-84bf5c887f from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:51Z" name: error-404-isvc-a1442-predictor.18c7687a1fa80ef4 namespace: kserve-ci-e2e-test resourceVersion: "21235" uid: 71a283b0-199a-44c3-92ca-a8835fae9aeb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a1442-predictor namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 72092aad-9e28-40fb-b8eb-bf889597df98 kind: Event lastTimestamp: "2026-07-31T15:11:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:11:36Z" name: error-404-isvc-a1442-predictor.18c7687d9e2b57d0 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: 4b8fe6c6-7009-4b7a-8b5a-4abf41c4a603 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a1442-predictor namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: 72092aad-9e28-40fb-b8eb-bf889597df98 kind: Event lastTimestamp: "2026-07-31T15:11:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:11:36Z" name: error-404-isvc-a1442-predictor.18c7687d9eb72884 namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: 368cf8f9-773a-4a8c-ad0c-a7f49ae78f0d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:11:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a1442-predictor namespace: kserve-ci-e2e-test resourceVersion: "21575" uid: 72092aad-9e28-40fb-b8eb-bf889597df98 kind: Event lastTimestamp: "2026-07-31T15:12:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:06Z" name: error-404-isvc-a1442-predictor.18c768811d616d31 namespace: kserve-ci-e2e-test resourceVersion: "22190" uid: 191a4319-98db-4aaf-aa8a-526f2d1f2fe5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:11:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a1442-predictor namespace: kserve-ci-e2e-test resourceVersion: "21575" uid: 72092aad-9e28-40fb-b8eb-bf889597df98 kind: Event lastTimestamp: "2026-07-31T15:12:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:11:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:06Z" name: error-404-isvc-a1442-predictor.18c768811dc30c85 namespace: kserve-ci-e2e-test resourceVersion: "22191" uid: 9c0d1217-b25c-4e45-ad95-15aa702162ac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a1442 namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 9e3107b8-49c2-4d89-b6fd-3df704ba5d7d kind: Event lastTimestamp: "2026-07-31T15:11:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:11:48Z" name: error-404-isvc-a1442.18c7687a1fa92683 namespace: kserve-ci-e2e-test resourceVersion: "21965" uid: e867a250-cb79-494f-8e17-1fc715dcf2aa reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a1442 namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 9e3107b8-49c2-4d89-b6fd-3df704ba5d7d kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-a1442": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a1442": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:10:51Z" name: error-404-isvc-a1442.18c7687a2033674a namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: de5b18f2-832b-475e-ae3d-e4a11c3ba62c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a1442 namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 9e3107b8-49c2-4d89-b6fd-3df704ba5d7d kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a1442": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:10:51Z" name: error-404-isvc-a1442.18c7687a20339beb namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: dc006b2b-cb44-46eb-8900-942c8ec7fc6d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a1442 namespace: kserve-ci-e2e-test resourceVersion: "21962" uid: 9e3107b8-49c2-4d89-b6fd-3df704ba5d7d kind: Event lastTimestamp: "2026-07-31T15:11:48Z" message: InferenceService [error-404-isvc-a1442] is Ready metadata: creationTimestamp: "2026-07-31T15:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:11:48Z" name: error-404-isvc-a1442.18c768879007a534 namespace: kserve-ci-e2e-test resourceVersion: "21963" uid: a409fa21-db3d-4251-850c-bf3aeb9d6f90 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:01:36.097936Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg namespace: kserve-ci-e2e-test resourceVersion: "16961" uid: cb998829-e9e8-4c7e-a2c1-b0ac62c98c74 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a9565-predictor-7689f7fc67-cvwzg to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:01:36Z" name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg.18c767f8eccaa7f3 namespace: kserve-ci-e2e-test resourceVersion: "16966" uid: e4c209d0-5701-42d0-b794-ee8f83aeb647 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg namespace: kserve-ci-e2e-test resourceVersion: "16971" uid: cb998829-e9e8-4c7e-a2c1-b0ac62c98c74 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:01:36Z" name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg.18c767f906d76365 namespace: kserve-ci-e2e-test resourceVersion: "16991" uid: 46c70060-84e5-410a-85fc-3fee9bda755c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: cb998829-e9e8-4c7e-a2c1-b0ac62c98c74 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg.18c767f9082ac4fe namespace: kserve-ci-e2e-test resourceVersion: "16994" uid: 7c82d2fc-5ea0-417a-beef-97308842d183 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: cb998829-e9e8-4c7e-a2c1-b0ac62c98c74 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg.18c767f90b074bac namespace: kserve-ci-e2e-test resourceVersion: "16996" uid: 133abe7d-5901-4953-985a-d9006cea1b2b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: cb998829-e9e8-4c7e-a2c1-b0ac62c98c74 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg.18c767f90b91b529 namespace: kserve-ci-e2e-test resourceVersion: "16997" uid: 869327f9-fda1-4d3b-9ccf-105dfd329813 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: cb998829-e9e8-4c7e-a2c1-b0ac62c98c74 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg.18c767f90b9cc0b6 namespace: kserve-ci-e2e-test resourceVersion: "16998" uid: f844854f-ca11-46b6-a54b-2245420c788a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: cb998829-e9e8-4c7e-a2c1-b0ac62c98c74 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg.18c767f90e759bae namespace: kserve-ci-e2e-test resourceVersion: "16999" uid: 0a87030e-b180-4764-bd8a-e4a4757ea648 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: cb998829-e9e8-4c7e-a2c1-b0ac62c98c74 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg.18c767f90eeabe8e namespace: kserve-ci-e2e-test resourceVersion: "17000" uid: dfc61975-1c89-4c1b-b4c8-b56d14233130 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:01:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: cb998829-e9e8-4c7e-a2c1-b0ac62c98c74 kind: Event lastTimestamp: "2026-07-31T15:02:14Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:02:14Z" name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg.18c767f97abb793c namespace: kserve-ci-e2e-test resourceVersion: "17363" uid: a890cd47-c359-4fc0-9a68-9bd1eb34e975 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: cb998829-e9e8-4c7e-a2c1-b0ac62c98c74 kind: Event lastTimestamp: "2026-07-31T15:10:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:50Z" name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg.18c7687a1a5a6b30 namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: cd07270b-0007-4133-aba9-ee8c2b7720ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: cb998829-e9e8-4c7e-a2c1-b0ac62c98c74 kind: Event lastTimestamp: "2026-07-31T15:10:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:50Z" name: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg.18c7687a1a5afc68 namespace: kserve-ci-e2e-test resourceVersion: "21214" uid: 6475b27f-4fab-4258-b61c-7550cbe0b564 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a9565-predictor-7689f7fc67 namespace: kserve-ci-e2e-test resourceVersion: "16952" uid: a1c4be36-b57c-4ad5-811d-16b9932fed1b kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: 'Created pod: error-404-isvc-a9565-predictor-7689f7fc67-cvwzg' metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:36Z" name: error-404-isvc-a9565-predictor-7689f7fc67.18c767f8ec2bbd04 namespace: kserve-ci-e2e-test resourceVersion: "16962" uid: 943027d1-cf23-4a7a-84c3-7a50b72c9223 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a9565-predictor namespace: kserve-ci-e2e-test resourceVersion: "16949" uid: 95adbc10-f024-486c-beb3-9c07d081d024 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: Scaled up replica set error-404-isvc-a9565-predictor-7689f7fc67 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:36Z" name: error-404-isvc-a9565-predictor.18c767f8ea8d9066 namespace: kserve-ci-e2e-test resourceVersion: "16954" uid: dd38fd33-42cb-4120-8a0b-ce1ccbe28f9c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:01:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9565-predictor namespace: kserve-ci-e2e-test resourceVersion: "16951" uid: 6bb66823-375e-45ad-9985-b0670713da5c kind: Event lastTimestamp: "2026-07-31T15:02:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:02:06Z" name: error-404-isvc-a9565-predictor.18c767fc6939d9d7 namespace: kserve-ci-e2e-test resourceVersion: "17297" uid: 7ceb3bfd-0b8f-4545-9a01-bf36b7396c05 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:01:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9565-predictor namespace: kserve-ci-e2e-test resourceVersion: "16951" uid: 6bb66823-375e-45ad-9985-b0670713da5c kind: Event lastTimestamp: "2026-07-31T15:02:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:01:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:02:06Z" name: error-404-isvc-a9565-predictor.18c767fc69a549be namespace: kserve-ci-e2e-test resourceVersion: "17298" uid: ecac0289-99b0-4329-8fcd-69ebfa804305 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:02:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9565-predictor namespace: kserve-ci-e2e-test resourceVersion: "17177" uid: 6bb66823-375e-45ad-9985-b0670713da5c kind: Event lastTimestamp: "2026-07-31T15:02:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:02:51Z" name: error-404-isvc-a9565-predictor.18c768036718b3ec namespace: kserve-ci-e2e-test resourceVersion: "17691" uid: 9f86962a-7b23-4389-bf1d-c1e8b143ff24 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:02:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9565-predictor namespace: kserve-ci-e2e-test resourceVersion: "17177" uid: 6bb66823-375e-45ad-9985-b0670713da5c kind: Event lastTimestamp: "2026-07-31T15:02:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:02:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:02:51Z" name: error-404-isvc-a9565-predictor.18c7680367942093 namespace: kserve-ci-e2e-test resourceVersion: "17692" uid: ecd36eea-b0ce-4ffd-9c2b-1677e8183615 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9565 namespace: kserve-ci-e2e-test resourceVersion: "16920" uid: 186f98c9-6e20-472b-a053-a0c66148a421 kind: Event lastTimestamp: "2026-07-31T15:02:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:02:24Z" name: error-404-isvc-a9565.18c767f8eadf8ec2 namespace: kserve-ci-e2e-test resourceVersion: "17455" uid: 18781c6b-ec5b-4310-8398-09be18d307e8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9565 namespace: kserve-ci-e2e-test resourceVersion: "16920" uid: 186f98c9-6e20-472b-a053-a0c66148a421 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: 'Failed to update status for InferenceService "error-404-isvc-a9565": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a9565": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:01:36Z" name: error-404-isvc-a9565.18c767f8eba4252c namespace: kserve-ci-e2e-test resourceVersion: "16960" uid: 3626f972-0f28-4533-941f-ba71c03b414a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9565 namespace: kserve-ci-e2e-test resourceVersion: "16920" uid: 186f98c9-6e20-472b-a053-a0c66148a421 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a9565": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:01:36Z" name: error-404-isvc-a9565.18c767f8eba475c6 namespace: kserve-ci-e2e-test resourceVersion: "16965" uid: c194a212-5e7c-4cce-a558-9eaef78295f7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:02:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9565 namespace: kserve-ci-e2e-test resourceVersion: "17453" uid: 186f98c9-6e20-472b-a053-a0c66148a421 kind: Event lastTimestamp: "2026-07-31T15:02:24Z" message: InferenceService [error-404-isvc-a9565] is Ready metadata: creationTimestamp: "2026-07-31T15:02:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:02:24Z" name: error-404-isvc-a9565.18c7680435ba2ac6 namespace: kserve-ci-e2e-test resourceVersion: "17454" uid: 7c4e3789-20fc-45cb-b948-42f7994f13d5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:20:20.304513Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e31d5-predictor-8545984775-n6bpj namespace: kserve-ci-e2e-test resourceVersion: "26207" uid: 5588be0d-175b-4bc7-a25e-91e4bfb55df4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e31d5-predictor-8545984775-n6bpj to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:20:20Z" name: error-404-isvc-e31d5-predictor-8545984775-n6bpj.18c768feacb99811 namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: 4e5f0500-822c-4df2-8bc5-6ef09468e024 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e31d5-predictor-8545984775-n6bpj namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 5588be0d-175b-4bc7-a25e-91e4bfb55df4 kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e31d5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:20Z" name: error-404-isvc-e31d5-predictor-8545984775-n6bpj.18c768feba3e2426 namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: 2ec17699-dd07-4f06-b174-ed116a541467 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e31d5-predictor-8545984775-n6bpj namespace: kserve-ci-e2e-test resourceVersion: "26224" uid: 5588be0d-175b-4bc7-a25e-91e4bfb55df4 kind: Event lastTimestamp: "2026-07-31T15:20:21Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:20:21Z" name: error-404-isvc-e31d5-predictor-8545984775-n6bpj.18c768feea2d4275 namespace: kserve-ci-e2e-test resourceVersion: "26255" uid: f916e6ac-e992-454c-aff3-52e7d4cca4e1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e31d5-predictor-8545984775-n6bpj namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 5588be0d-175b-4bc7-a25e-91e4bfb55df4 kind: Event lastTimestamp: "2026-07-31T15:20:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:21Z" name: error-404-isvc-e31d5-predictor-8545984775-n6bpj.18c768feeb935c5c namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: a33627af-769a-483d-ac5f-ee03638b3335 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e31d5-predictor-8545984775-n6bpj namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 5588be0d-175b-4bc7-a25e-91e4bfb55df4 kind: Event lastTimestamp: "2026-07-31T15:20:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:21Z" name: error-404-isvc-e31d5-predictor-8545984775-n6bpj.18c768feee91884f namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 116d903e-0001-45cd-815b-4aefa2d2b7ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e31d5-predictor-8545984775-n6bpj namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 5588be0d-175b-4bc7-a25e-91e4bfb55df4 kind: Event lastTimestamp: "2026-07-31T15:20:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:21Z" name: error-404-isvc-e31d5-predictor-8545984775-n6bpj.18c768feef0dc83c namespace: kserve-ci-e2e-test resourceVersion: "26259" uid: b489369f-fe39-4b3d-96bd-0b473d08c0b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e31d5-predictor-8545984775-n6bpj namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 5588be0d-175b-4bc7-a25e-91e4bfb55df4 kind: Event lastTimestamp: "2026-07-31T15:20:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:21Z" name: error-404-isvc-e31d5-predictor-8545984775-n6bpj.18c768feef1814df namespace: kserve-ci-e2e-test resourceVersion: "26260" uid: 7a1a260a-878b-479b-992b-ad69529bf1a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e31d5-predictor-8545984775-n6bpj namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 5588be0d-175b-4bc7-a25e-91e4bfb55df4 kind: Event lastTimestamp: "2026-07-31T15:20:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:21Z" name: error-404-isvc-e31d5-predictor-8545984775-n6bpj.18c768fef25b05a9 namespace: kserve-ci-e2e-test resourceVersion: "26261" uid: abf3a54c-3eed-4605-9bb3-dcf71c7c3073 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e31d5-predictor-8545984775-n6bpj namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 5588be0d-175b-4bc7-a25e-91e4bfb55df4 kind: Event lastTimestamp: "2026-07-31T15:20:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:21Z" name: error-404-isvc-e31d5-predictor-8545984775-n6bpj.18c768fef2d3966a namespace: kserve-ci-e2e-test resourceVersion: "26262" uid: 5ed91273-7ae8-4436-817a-63ffe6930fff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:20:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e31d5-predictor-8545984775-n6bpj namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 5588be0d-175b-4bc7-a25e-91e4bfb55df4 kind: Event lastTimestamp: "2026-07-31T15:20:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:57Z" name: error-404-isvc-e31d5-predictor-8545984775-n6bpj.18c768ff014a365d namespace: kserve-ci-e2e-test resourceVersion: "26556" uid: e00fb108-69c4-4ad8-86f9-14cfec5fc50a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e31d5-predictor-8545984775-n6bpj namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 5588be0d-175b-4bc7-a25e-91e4bfb55df4 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: error-404-isvc-e31d5-predictor-8545984775-n6bpj.18c7690f266f766c namespace: kserve-ci-e2e-test resourceVersion: "27082" uid: 8e18ba92-9962-4908-92d8-f272f9228a19 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e31d5-predictor-8545984775-n6bpj namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 5588be0d-175b-4bc7-a25e-91e4bfb55df4 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: error-404-isvc-e31d5-predictor-8545984775-n6bpj.18c7690f266feb12 namespace: kserve-ci-e2e-test resourceVersion: "27083" uid: f0655272-d210-4775-9e13-a67d48828502 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e31d5-predictor-8545984775-n6bpj namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 5588be0d-175b-4bc7-a25e-91e4bfb55df4 kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: error-404-isvc-e31d5-predictor-8545984775-n6bpj.18c7690f898645fc namespace: kserve-ci-e2e-test resourceVersion: "27134" uid: 18a1a639-142e-4579-87ef-ee3d58d7a015 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e31d5-predictor-8545984775 namespace: kserve-ci-e2e-test resourceVersion: "26200" uid: 92dc80b0-2f5d-49bc-9390-e54ec7f82eae kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: 'Created pod: error-404-isvc-e31d5-predictor-8545984775-n6bpj' metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:20:20Z" name: error-404-isvc-e31d5-predictor-8545984775.18c768feac30bca4 namespace: kserve-ci-e2e-test resourceVersion: "26209" uid: ff0e58e1-5d85-4a38-9483-e02fa883c2e9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e31d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26197" uid: cea569e8-d233-4a97-8ee1-d21d003eb80d kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: Scaled up replica set error-404-isvc-e31d5-predictor-8545984775 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:20:20Z" name: error-404-isvc-e31d5-predictor.18c768feaa502a34 namespace: kserve-ci-e2e-test resourceVersion: "26201" uid: 77fd7932-0cb7-476e-b19c-9c1babb9d150 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:20:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e31d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26199" uid: fe389022-d62e-4aba-aad2-31d9ed592de5 kind: Event lastTimestamp: "2026-07-31T15:20:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:20:50Z" name: error-404-isvc-e31d5-predictor.18c7690228c26881 namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: 3bba5365-b3a1-4b5a-8e71-702ab884d6c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:20:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e31d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26199" uid: fe389022-d62e-4aba-aad2-31d9ed592de5 kind: Event lastTimestamp: "2026-07-31T15:20:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:20:50Z" name: error-404-isvc-e31d5-predictor.18c76902290ce995 namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: 6f097a6f-a46c-42be-83e1-e8904a22c418 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:21:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e31d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26387" uid: fe389022-d62e-4aba-aad2-31d9ed592de5 kind: Event lastTimestamp: "2026-07-31T15:21:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:21:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:20Z" name: error-404-isvc-e31d5-predictor.18c7690926a23cf8 namespace: kserve-ci-e2e-test resourceVersion: "26753" uid: e13418f0-44de-495d-997b-76895901627d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:21:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e31d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26387" uid: fe389022-d62e-4aba-aad2-31d9ed592de5 kind: Event lastTimestamp: "2026-07-31T15:21:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:21:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:20Z" name: error-404-isvc-e31d5-predictor.18c769092704d181 namespace: kserve-ci-e2e-test resourceVersion: "26755" uid: 802694d4-6c2e-477b-8bf3-da5bd40bbad5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e31d5 namespace: kserve-ci-e2e-test resourceVersion: "26181" uid: 700fedfe-ca9e-4561-a3f8-918620c2023c kind: Event lastTimestamp: "2026-07-31T15:21:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:21:07Z" name: error-404-isvc-e31d5.18c768feaa4c59f5 namespace: kserve-ci-e2e-test resourceVersion: "26650" uid: d0a54209-16f0-435d-a70a-4f6f5e0e66eb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e31d5 namespace: kserve-ci-e2e-test resourceVersion: "26181" uid: 700fedfe-ca9e-4561-a3f8-918620c2023c kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: 'Failed to update status for InferenceService "error-404-isvc-e31d5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e31d5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:20:20Z" name: error-404-isvc-e31d5.18c768feab08fc9b namespace: kserve-ci-e2e-test resourceVersion: "26204" uid: 6c6e2304-c0e4-4109-9a10-ce72068c1657 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e31d5 namespace: kserve-ci-e2e-test resourceVersion: "26181" uid: 700fedfe-ca9e-4561-a3f8-918620c2023c kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e31d5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:20:20Z" name: error-404-isvc-e31d5.18c768feab094bf7 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: dbf620c2-d4c1-4cfc-92b7-ffca5ed6bf6d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e31d5 namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 700fedfe-ca9e-4561-a3f8-918620c2023c kind: Event lastTimestamp: "2026-07-31T15:21:07Z" message: InferenceService [error-404-isvc-e31d5] is Ready metadata: creationTimestamp: "2026-07-31T15:21:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:21:07Z" name: error-404-isvc-e31d5.18c76909bd24f549 namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: a1d5a821-62ed-4e6b-bd80-b1225ecb9447 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T14:59:06.150847Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15208" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e9535-predictor-54f44fc89c-p24vc to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T14:59:06Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767d6033fa0d9 namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: b745b41c-f4b2-4d3d-a46c-de45182b9c01 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15218" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Add eth0 [10.134.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T14:59:06Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767d61d846e1d namespace: kserve-ci-e2e-test resourceVersion: "15263" uid: ac188fc5-8047-43a6-954e-0152a1a0b310 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1833" metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:06Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767d61f8fa918 namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: 538d43cc-714a-4418-9efa-c5a6f60b8767 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: "2026-07-31T14:59:22Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1833" in 15.834s (15.834s including waiting). Image size: 1333970528 bytes.' metadata: creationTimestamp: "2026-07-31T14:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:22Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767d9cf5aa82f namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 66648be2-50a7-4828-8e28-55ac574301b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: "2026-07-31T14:59:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T14:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:22Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767d9d3338311 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: e30c82fd-ddaf-4966-a78e-fbaf35957e75 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: "2026-07-31T14:59:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T14:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:22Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767d9d3b79899 namespace: kserve-ci-e2e-test resourceVersion: "15450" uid: c35d1737-3c3a-4095-ba60-267fa7f45df9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: "2026-07-31T14:59:22Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-31T14:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:22Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767d9d3c3310d namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: 7c118faa-d775-446d-96c6-85e8b1e811bc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: "2026-07-31T14:59:25Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.795s (2.795s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-31T14:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:25Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767da7a6991c2 namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: da477f39-20a4-4aa1-9a1b-1f8136417dbf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: "2026-07-31T14:59:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T14:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:25Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767da7ee0abbe namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: 834a25ac-4cd1-4ef4-a2ff-92761568f506 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: "2026-07-31T14:59:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T14:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:25Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767da7f5addb5 namespace: kserve-ci-e2e-test resourceVersion: "15477" uid: c4f88e16-fb5f-4cdf-8a30-178b7e40e23f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T14:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: "2026-07-31T15:00:41Z" message: 'Readiness probe failed: dial tcp 10.134.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T14:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:41Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767daa107cafc namespace: kserve-ci-e2e-test resourceVersion: "16323" uid: 963014af-691c-4c80-9b27-6b757156a46c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767ebf30cf93f namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 2040ba1d-e420-4216-970e-0c5ae6f4da25 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767ebf30e0c68 namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: 41b2e6b7-0763-48bc-89b0-f443527c220d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 43d60a8c-0e8f-4a59-ad42-c1cbf1819e18 kind: Event lastTimestamp: "2026-07-31T15:00:41Z" message: 'Readiness probe failed: Get "https://10.134.0.18:8643/healthz": dial tcp 10.134.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:41Z" name: error-404-isvc-e9535-predictor-54f44fc89c-p24vc.18c767ec53326d0c namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 86d9cc36-b4a4-45d4-ac23-47bb2ccd0abe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e9535-predictor-54f44fc89c namespace: kserve-ci-e2e-test resourceVersion: "15201" uid: e7bc2f22-600c-475c-b158-b605d788086c kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: 'Created pod: error-404-isvc-e9535-predictor-54f44fc89c-p24vc' metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T14:59:06Z" name: error-404-isvc-e9535-predictor-54f44fc89c.18c767d6025d040a namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 1c47c043-0e44-48f3-936b-b894258b80ea reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e9535-predictor namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: 68e6d740-b8c9-4dde-ba54-39c693502d79 kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Scaled up replica set error-404-isvc-e9535-predictor-54f44fc89c from 0 to 1 metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T14:59:06Z" name: error-404-isvc-e9535-predictor.18c767d60012fa9b namespace: kserve-ci-e2e-test resourceVersion: "15204" uid: 0e2df7fc-3241-447c-89b0-8c3ad988a820 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T14:59:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e9535-predictor namespace: kserve-ci-e2e-test resourceVersion: "15200" uid: 31fdf5df-b38e-4e0b-9c15-db2e31123dd7 kind: Event lastTimestamp: "2026-07-31T15:00:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T14:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:06Z" name: error-404-isvc-e9535-predictor.18c767d97e781cc5 namespace: kserve-ci-e2e-test resourceVersion: "15836" uid: 30dbec06-a5e0-4154-a957-a7e93cfa3521 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T14:59:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e9535-predictor namespace: kserve-ci-e2e-test resourceVersion: "15200" uid: 31fdf5df-b38e-4e0b-9c15-db2e31123dd7 kind: Event lastTimestamp: "2026-07-31T15:00:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T14:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:06Z" name: error-404-isvc-e9535-predictor.18c767d97ef8e191 namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: 9d35a2b2-d376-40b4-a815-dbc4d05b2f19 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T14:59:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e9535-predictor namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: 31fdf5df-b38e-4e0b-9c15-db2e31123dd7 kind: Event lastTimestamp: "2026-07-31T15:00:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T14:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:36Z" name: error-404-isvc-e9535-predictor.18c767e07c735936 namespace: kserve-ci-e2e-test resourceVersion: "16131" uid: bca449b9-2429-4d47-a78d-30d3b3578bc7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T14:59:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e9535-predictor namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: 31fdf5df-b38e-4e0b-9c15-db2e31123dd7 kind: Event lastTimestamp: "2026-07-31T15:00:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T14:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:36Z" name: error-404-isvc-e9535-predictor.18c767e07ced8aca namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: 6778296b-5953-4dc5-af56-666102c125de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e9535 namespace: kserve-ci-e2e-test resourceVersion: "15184" uid: 0596def3-6b79-4053-9de9-be4b17b5c873 kind: Event lastTimestamp: "2026-07-31T15:00:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:12Z" name: error-404-isvc-e9535.18c767d60045330f namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: 0947779f-5891-4fc4-a8c2-cc4bad7fed91 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e9535 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: 0596def3-6b79-4053-9de9-be4b17b5c873 kind: Event lastTimestamp: "2026-07-31T15:00:12Z" message: InferenceService [error-404-isvc-e9535] is Ready metadata: creationTimestamp: "2026-07-31T15:00:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:12Z" name: error-404-isvc-e9535.18c767e55be8b217 namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: 6ced8beb-e598-4096-9849-e430d525047e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:21:30.938343Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp namespace: kserve-ci-e2e-test resourceVersion: "27033" uid: 5fa64ced-8e39-4ede-9456-34d853453666 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:21:30Z" name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp.18c7690f1ed45892 namespace: kserve-ci-e2e-test resourceVersion: "27037" uid: b037a705-fdae-4566-9129-caa6dbf12138 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: 5fa64ced-8e39-4ede-9456-34d853453666 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-faf8d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp.18c7690f2d6331fb namespace: kserve-ci-e2e-test resourceVersion: "27097" uid: 7976c6ad-695c-4dea-8588-46d4dfc421cd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp namespace: kserve-ci-e2e-test resourceVersion: "27045" uid: 5fa64ced-8e39-4ede-9456-34d853453666 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:21:31Z" name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp.18c7690f5d1e6cb8 namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: 489237b9-dade-4cca-b5f6-05d5a889ebcf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: 5fa64ced-8e39-4ede-9456-34d853453666 kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp.18c7690f5f02a949 namespace: kserve-ci-e2e-test resourceVersion: "27119" uid: cc1453a2-914d-4275-8644-0effefc19bc5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: 5fa64ced-8e39-4ede-9456-34d853453666 kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp.18c7690f61353cb0 namespace: kserve-ci-e2e-test resourceVersion: "27124" uid: c1e897aa-3fea-426a-82d6-1c392197ec1c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: 5fa64ced-8e39-4ede-9456-34d853453666 kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp.18c7690f61aee14f namespace: kserve-ci-e2e-test resourceVersion: "27125" uid: 728aff99-b896-4232-b403-744ca6fbfcd9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: 5fa64ced-8e39-4ede-9456-34d853453666 kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp.18c7690f61b83d11 namespace: kserve-ci-e2e-test resourceVersion: "27126" uid: bbdb70cf-523d-4b55-8c05-66b3db7d1076 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: 5fa64ced-8e39-4ede-9456-34d853453666 kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp.18c7690f64bc562f namespace: kserve-ci-e2e-test resourceVersion: "27127" uid: 9880d1dd-3f2e-440c-8d42-0b92d6d5e48d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: 5fa64ced-8e39-4ede-9456-34d853453666 kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp.18c7690f653d7c0e namespace: kserve-ci-e2e-test resourceVersion: "27128" uid: 96f6ee7e-e0f9-4305-b97e-5486c1c5fd13 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T15:21:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: 5fa64ced-8e39-4ede-9456-34d853453666 kind: Event lastTimestamp: "2026-07-31T15:22:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:21:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:22:20Z" name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp.18c7690fd6b05a84 namespace: kserve-ci-e2e-test resourceVersion: "27570" uid: 04f84fac-7ba3-4d13-9fae-cd503bddc2e2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:30:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: 5fa64ced-8e39-4ede-9456-34d853453666 kind: Event lastTimestamp: "2026-07-31T15:30:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:30:55Z" name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp.18c76992935973d0 namespace: kserve-ci-e2e-test resourceVersion: "31286" uid: 7ae1c39b-2397-400b-ace2-7571e1a0b025 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:30:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp namespace: kserve-ci-e2e-test resourceVersion: "27035" uid: 5fa64ced-8e39-4ede-9456-34d853453666 kind: Event lastTimestamp: "2026-07-31T15:30:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:30:55Z" name: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp.18c76992935a074d namespace: kserve-ci-e2e-test resourceVersion: "31288" uid: cd286098-77eb-4a81-9bda-b365bac9b6d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-faf8d-predictor-6f9fc8c58f namespace: kserve-ci-e2e-test resourceVersion: "27025" uid: d0d7931a-3a3a-43b2-b658-a20a13f65908 kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: 'Created pod: error-404-isvc-faf8d-predictor-6f9fc8c58f-8vbhp' metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:30Z" name: error-404-isvc-faf8d-predictor-6f9fc8c58f.18c7690f1e27f75a namespace: kserve-ci-e2e-test resourceVersion: "27034" uid: daf8218a-985b-46b9-9d3f-6136c12b7ba7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-faf8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "27022" uid: 703d6a2f-96d7-4f05-850b-2cf6755d070f kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Scaled up replica set error-404-isvc-faf8d-predictor-6f9fc8c58f from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:30Z" name: error-404-isvc-faf8d-predictor.18c7690f1c59f0c2 namespace: kserve-ci-e2e-test resourceVersion: "27026" uid: 8ef3e70b-c47b-4056-aa15-e9bd550c56c0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-faf8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "27024" uid: e590f409-9595-45dc-b161-96ce5134ab9a kind: Event lastTimestamp: "2026-07-31T15:21:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:21:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:45Z" name: error-404-isvc-faf8d-predictor.18c769129ad54636 namespace: kserve-ci-e2e-test resourceVersion: "27276" uid: 75eb8f1c-c081-4483-9d0c-8e909f79b09c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-faf8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "27024" uid: e590f409-9595-45dc-b161-96ce5134ab9a kind: Event lastTimestamp: "2026-07-31T15:21:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:21:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:45Z" name: error-404-isvc-faf8d-predictor.18c769129b4123cd namespace: kserve-ci-e2e-test resourceVersion: "27278" uid: d20bbed5-bf98-4243-87f5-5b703fb8fa26 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:22:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-faf8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "27277" uid: e590f409-9595-45dc-b161-96ce5134ab9a kind: Event lastTimestamp: "2026-07-31T15:23:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:22:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:23:00Z" name: error-404-isvc-faf8d-predictor.18c769161a183ba1 namespace: kserve-ci-e2e-test resourceVersion: "27982" uid: 4aacd77f-c05a-41e1-bf76-692a025915e7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:22:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-faf8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "27277" uid: e590f409-9595-45dc-b161-96ce5134ab9a kind: Event lastTimestamp: "2026-07-31T15:23:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:22:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:23:00Z" name: error-404-isvc-faf8d-predictor.18c769161ad24f78 namespace: kserve-ci-e2e-test resourceVersion: "27983" uid: 52491502-0f2f-4073-8ad7-9acf5d85b25b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-faf8d namespace: kserve-ci-e2e-test resourceVersion: "27009" uid: 8b3707b1-d548-4d03-a9e2-dcb93ca6359a kind: Event lastTimestamp: "2026-07-31T15:22:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:22:30Z" name: error-404-isvc-faf8d.18c7690f1c5fe6ec namespace: kserve-ci-e2e-test resourceVersion: "27666" uid: 41fa1028-1d1f-4c74-b447-946d6c6e0622 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-faf8d namespace: kserve-ci-e2e-test resourceVersion: "27664" uid: 8b3707b1-d548-4d03-a9e2-dcb93ca6359a kind: Event lastTimestamp: "2026-07-31T15:22:30Z" message: InferenceService [error-404-isvc-faf8d] is Ready metadata: creationTimestamp: "2026-07-31T15:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:22:30Z" name: error-404-isvc-faf8d.18c7691ce6f8ee72 namespace: kserve-ci-e2e-test resourceVersion: "27665" uid: 2cb674fc-427b-460a-a821-a5566b6b021c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T14:59:05.931067Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15156" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh to ip-10-0-143-167.ec2.internal metadata: creationTimestamp: "2026-07-31T14:59:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T14:59:05Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767d5f625ff6b namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 1d83e91a-ec6e-4452-9ba9-6aa4d7f402fa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T14:59:06Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767d60fe5aa3b namespace: kserve-ci-e2e-test resourceVersion: "15237" uid: 95dfccce-2092-4f77-bec8-bfbe3a06ec1d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9f3ee656918f8773cccaed053f1f1333c434d42ed980ad7a0ad04ab3b2c3253b" already present on machine metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:06Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767d611d0248d namespace: kserve-ci-e2e-test resourceVersion: "15239" uid: 1414c3d4-dcab-47a9-ba94-34ce33da364e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:06Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767d6159288aa namespace: kserve-ci-e2e-test resourceVersion: "15248" uid: 25a60951-da5b-4680-afd9-dbaf22a97930 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:06Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767d61660bf7d namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 5a443dfd-8e8a-4c41-a3ff-b33c8d272e8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T14:59:10Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1833" metadata: creationTimestamp: "2026-07-31T14:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:10Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767d71806c861 namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 6706766b-6010-428e-ac88-4b4487338140 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T14:59:38Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1833" in 27.604s (27.604s including waiting). Image size: 1557910953 bytes.' metadata: creationTimestamp: "2026-07-31T14:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:38Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767dd85651b2b namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 24c59e4e-c561-47da-89a7-3e91cb1c31bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T14:59:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T14:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:38Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767dd97752fe1 namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: ec9ac78b-35a7-4e40-a6cf-5efacfa1a9d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T14:59:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T14:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:38Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767dd9888a417 namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: b0e4faed-81e7-4431-a3b6-ddcd79aefe1b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T14:59:38Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-31T14:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:38Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767dd989439e9 namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: 187df8bf-09a0-4123-abae-660a83bfcca6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T14:59:41Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.824s (2.825s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-31T14:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:41Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767de40f6d090 namespace: kserve-ci-e2e-test resourceVersion: "15619" uid: 282a319c-934b-4e5a-b31c-22873d73da86 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T14:59:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T14:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:41Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767de45481bb6 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 03ff7a9f-cd6e-4265-b621-f3122bc536df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T14:59:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T14:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:41Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767de45c51370 namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: 09303795-e440-44ff-80d6-8276c4c669ee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-31T14:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T15:01:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T14:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:38Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767de5cbc4e9f namespace: kserve-ci-e2e-test resourceVersion: "17022" uid: 27af8cf0-049f-4a60-aa55-d2450148ac92 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T15:01:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:35Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767f8e569b4ea namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: 80a5eecf-f197-485f-9f58-a4c367889ba8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T15:01:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:35Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767f8e56a22c5 namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 859b349e-231b-48b8-a04f-f2dfbfaf5d4e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 177705e1-c723-4ef0-ab4c-0587a4facb6a kind: Event lastTimestamp: "2026-07-31T15:01:38Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:38Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh.18c767f95f4f2082 namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: fa3591a2-f5e8-4f98-bfe5-d768da806af3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-7dc5b78b48 namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: ed4848e6-cacb-4b0c-8bfa-9bca5cc7decd kind: Event lastTimestamp: "2026-07-31T14:59:05Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-7dc5b78b48-hpbgh' metadata: creationTimestamp: "2026-07-31T14:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T14:59:05Z" name: isvc-sklearn-graph-1-predictor-7dc5b78b48.18c767d5f55f323f namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 88139cb0-ec1a-49b1-8fc9-aeeda1390514 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: 5b8e3406-7a1d-41c3-808a-c31c7ea67608 kind: Event lastTimestamp: "2026-07-31T14:59:05Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-7dc5b78b48 from 0 to 1 metadata: creationTimestamp: "2026-07-31T14:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T14:59:05Z" name: isvc-sklearn-graph-1-predictor.18c767d5f2795ecf namespace: kserve-ci-e2e-test resourceVersion: "15149" uid: 1aa9b4c9-ea66-42d9-b8d1-085b85138eba reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T14:59:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: fa2f5e6f-cdbb-43f1-ac38-fd10dc4587a4 kind: Event lastTimestamp: "2026-07-31T15:00:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T14:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:20Z" name: isvc-sklearn-graph-1-predictor.18c767d971016a29 namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: a1a399d0-3f47-4974-a5d6-6b6e3a534cd7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T14:59:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: fa2f5e6f-cdbb-43f1-ac38-fd10dc4587a4 kind: Event lastTimestamp: "2026-07-31T15:00:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T14:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:20Z" name: isvc-sklearn-graph-1-predictor.18c767d9716cd127 namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 001fdfbe-b034-457a-8892-a1c5d59abb5a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T15:00:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: fa2f5e6f-cdbb-43f1-ac38-fd10dc4587a4 kind: Event lastTimestamp: "2026-07-31T15:01:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:20Z" name: isvc-sklearn-graph-1-predictor.18c767eaeb354590 namespace: kserve-ci-e2e-test resourceVersion: "16674" uid: 0c9098f1-0b5f-46ef-8fc9-3107afe9afd6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T15:00:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: fa2f5e6f-cdbb-43f1-ac38-fd10dc4587a4 kind: Event lastTimestamp: "2026-07-31T15:01:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:00:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:20Z" name: isvc-sklearn-graph-1-predictor.18c767eaeb9b7d51 namespace: kserve-ci-e2e-test resourceVersion: "16675" uid: b802a949-052f-45a1-b5db-8e6082e84af6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-31T14:59:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 303479ed-6760-4923-87da-29d3117efa4e kind: Event lastTimestamp: "2026-07-31T15:00:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T14:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:58Z" name: isvc-sklearn-graph-1.18c767d5f27d60bb namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: f07415d3-8722-4281-9336-9852477b1ef5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: 303479ed-6760-4923-87da-29d3117efa4e kind: Event lastTimestamp: "2026-07-31T15:00:58Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-31T15:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:58Z" name: isvc-sklearn-graph-1.18c767f01481c7aa namespace: kserve-ci-e2e-test resourceVersion: "16478" uid: f42b4b4f-2de8-4280-a615-af209d281b99 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T14:59:06.446382Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15243" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj to ip-10-0-143-167.ec2.internal metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T14:59:06Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767d614dd29f8 namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: 2252b300-cfa1-4ef8-b659-6a83ca66bce0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T14:59:06Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767d62fbac45b namespace: kserve-ci-e2e-test resourceVersion: "15283" uid: 4ad278d0-3112-4926-918e-43f6906cf21e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9f3ee656918f8773cccaed053f1f1333c434d42ed980ad7a0ad04ab3b2c3253b" already present on machine metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:06Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767d631c24850 namespace: kserve-ci-e2e-test resourceVersion: "15288" uid: 419a9bed-6df6-4c46-93db-764708f9b255 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:06Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767d6348611a2 namespace: kserve-ci-e2e-test resourceVersion: "15291" uid: 42c09ed0-44f3-4528-8b4d-9c27c7fff3aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:06Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767d63508811a namespace: kserve-ci-e2e-test resourceVersion: "15293" uid: 97921ea5-653d-4bd7-878d-74576e49e6fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T14:59:10Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1833" metadata: creationTimestamp: "2026-07-31T14:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:10Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767d717f01e60 namespace: kserve-ci-e2e-test resourceVersion: "15334" uid: ec112b7f-ca09-4ccf-a5fe-2a33a0ec6bc7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T14:59:38Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1833" in 28.041s (28.041s including waiting). Image size: 1557910953 bytes.' metadata: creationTimestamp: "2026-07-31T14:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:38Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767dd9f5c6c56 namespace: kserve-ci-e2e-test resourceVersion: "15599" uid: 5e2931d6-f733-4156-bae2-3e6fc6292c73 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T14:59:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T14:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:38Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767dda5f7ecb2 namespace: kserve-ci-e2e-test resourceVersion: "15600" uid: 48067336-6aee-4798-9169-4bcbd7031200 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T14:59:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T14:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:38Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767dda7102c1c namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: 7bfbf871-a522-4a17-a03d-c53e5f2813a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T14:59:38Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-31T14:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:38Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767dda71e6ff1 namespace: kserve-ci-e2e-test resourceVersion: "15602" uid: 83496e78-310f-4e4c-a7f6-a482163584dc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T14:59:41Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.59s (2.59s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-31T14:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:41Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767de418acdfc namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: 4a58d972-4669-4e69-b528-b7b858fd929f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T14:59:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T14:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:41Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767de45e49c50 namespace: kserve-ci-e2e-test resourceVersion: "15624" uid: 912784b9-0ae6-471c-be4b-f096d7d162a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T14:59:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T14:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:41Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767de465b9fcf namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 42b6a3fa-2f65-45ae-be42-316738b509d5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-31T14:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T15:01:39Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T14:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:39Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767de5c9aecaf namespace: kserve-ci-e2e-test resourceVersion: "17033" uid: 00a059ff-3099-4fa2-a363-b9f6c1afb79f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T15:01:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:35Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767f8e130b56f namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: dda5e2bf-f4be-47b6-8019-ee7de1d004a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T15:01:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:35Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767f8e1312ce7 namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: a39874a1-4e53-4b54-96fa-c1f0c1557133 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: 9d1f1ca3-d3a9-46f1-98e5-535a4a6e362c kind: Event lastTimestamp: "2026-07-31T15:01:39Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:39Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj.18c767f99b1f850d namespace: kserve-ci-e2e-test resourceVersion: "17032" uid: 2d54d460-57b2-4a25-be4b-e17111a7dbea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-54ff6676d8 namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: 490e7767-ca93-4107-bf91-356fde1770bf kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-54ff6676d8-8vwrj' metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T14:59:06Z" name: isvc-sklearn-graph-2-predictor-54ff6676d8.18c767d61472b166 namespace: kserve-ci-e2e-test resourceVersion: "15245" uid: 189769c6-7147-4aee-aad6-ee98811c525e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15227" uid: 4227b243-20ee-4d1d-872d-09462f3bc85d kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-54ff6676d8 from 0 to 1 metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T14:59:06Z" name: isvc-sklearn-graph-2-predictor.18c767d60c40ecc9 namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: 2a31c37f-9868-4d20-9aa6-e0f893015174 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T14:59:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: 7367e2a9-4942-4216-8dd9-5873afe461cb kind: Event lastTimestamp: "2026-07-31T15:00:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T14:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:21Z" name: isvc-sklearn-graph-2-predictor.18c767d98ac9246e namespace: kserve-ci-e2e-test resourceVersion: "15982" uid: d8891360-4e01-4fcd-b6e9-c916e04a6ef3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T14:59:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: 7367e2a9-4942-4216-8dd9-5873afe461cb kind: Event lastTimestamp: "2026-07-31T15:00:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T14:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:21Z" name: isvc-sklearn-graph-2-predictor.18c767d98b308fe8 namespace: kserve-ci-e2e-test resourceVersion: "15983" uid: 2db5c046-710c-474a-b2fa-d86a0af28934 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:00:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: 7367e2a9-4942-4216-8dd9-5873afe461cb kind: Event lastTimestamp: "2026-07-31T15:01:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:21Z" name: isvc-sklearn-graph-2-predictor.18c767e407afe9ea namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: 399f7706-3635-420a-8f1a-34b784631fd2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:00:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: 7367e2a9-4942-4216-8dd9-5873afe461cb kind: Event lastTimestamp: "2026-07-31T15:01:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:21Z" name: isvc-sklearn-graph-2-predictor.18c767e40860d792 namespace: kserve-ci-e2e-test resourceVersion: "16680" uid: 1e0e9c7c-80e7-4209-97d2-9827627b299b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: b9e697bd-d916-4a2a-9c40-de3096c4f1ee kind: Event lastTimestamp: "2026-07-31T15:00:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:59Z" name: isvc-sklearn-graph-2.18c767d60c780820 namespace: kserve-ci-e2e-test resourceVersion: "16497" uid: fc60e4bb-9e56-4b0a-a4e8-58565deff432 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: b9e697bd-d916-4a2a-9c40-de3096c4f1ee kind: Event lastTimestamp: "2026-07-31T15:00:59Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-31T15:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:59Z" name: isvc-sklearn-graph-2.18c767f050d41309 namespace: kserve-ci-e2e-test resourceVersion: "16496" uid: d1267c6c-85fd-4955-9937-5f3b5fa93395 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T14:59:06.056450Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-zcbs6 to ip-10-0-143-167.ec2.internal metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T14:59:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767d5fd9f4b27 namespace: kserve-ci-e2e-test resourceVersion: "15191" uid: e49de22c-bf49-48d8-9035-fb38fa2f31d1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15202" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T14:59:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767d617bce593 namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: 4c2641c1-5f1d-4257-9154-e980d07b1e91 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:9f3ee656918f8773cccaed053f1f1333c434d42ed980ad7a0ad04ab3b2c3253b" already present on machine metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767d6199992f2 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: df9a7c8c-535d-4f04-bfd2-65c52aa4a702 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767d61c81a5b6 namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: fe49354a-2aa5-422d-ada5-3607009552f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:06Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767d61cfe44a5 namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: 5589711f-97e7-4e5b-9bdc-025c555ef438 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T14:59:10Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-31T14:59:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767d718245beb namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: 3c843144-166d-4f00-aba2-35fb44cd525d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T14:59:38Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 27.873s (27.873s including waiting). Image size: 1307355898 bytes.' metadata: creationTimestamp: "2026-07-31T14:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767dd958efc20 namespace: kserve-ci-e2e-test resourceVersion: "15591" uid: cd1158be-f23f-4a36-b3f7-488e481e4aa7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T14:59:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T14:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767dd9c8b1406 namespace: kserve-ci-e2e-test resourceVersion: "15596" uid: 17657eae-18b3-436c-974d-5d71065880ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T14:59:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T14:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767dd9dcc5224 namespace: kserve-ci-e2e-test resourceVersion: "15597" uid: c1b9b5af-c4f2-4de4-9e47-533a964a99f0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T14:59:38Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-31T14:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767dd9ddaafe9 namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: 1455017f-789b-4801-acee-c45573c794e0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T14:59:41Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.778s (2.778s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-31T14:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767de43744f46 namespace: kserve-ci-e2e-test resourceVersion: "15621" uid: 0f6c44ad-21e5-4e83-a17a-4a1cc12e750c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T14:59:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T14:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767de463cca94 namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: c93a3448-944b-4c55-b8d1-393d19f1fc5e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T14:59:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T14:59:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767de46c08d85 namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: 6be49f03-f1ee-45c7-8172-8755c6b855bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-31T14:59:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T15:01:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T14:59:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767de5cddeea2 namespace: kserve-ci-e2e-test resourceVersion: "17021" uid: a63a14fa-5f19-4619-973b-dc8b00f1c399 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767f8e8d7c8aa namespace: kserve-ci-e2e-test resourceVersion: "16943" uid: fcf11893-aabf-4d55-94a1-27cb2f5e5adf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767f8e8d87e44 namespace: kserve-ci-e2e-test resourceVersion: "16945" uid: 8d41a453-57a7-4403-94ef-5e2df5ec54f2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 9a84c80b-8b9d-456d-bc0b-d9e71bb86dca kind: Event lastTimestamp: "2026-07-31T15:01:38Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:01:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-zcbs6.18c767f95f4dff2e namespace: kserve-ci-e2e-test resourceVersion: "17019" uid: ca4ed591-b57e-40f5-90f2-db4f33b4311e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-167.ec2.internal source: component: kubelet host: ip-10-0-143-167.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: 6fd09363-1f20-412c-b75a-d160ccd88993 kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-zcbs6' metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T14:59:06Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c767d5fcd23ae4 namespace: kserve-ci-e2e-test resourceVersion: "15187" uid: a54bf33e-bae1-42a0-92fe-fb8ef8273bbd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15173" uid: 7a4cf2e5-c434-40c5-a76c-10ae8b44ad5e kind: Event lastTimestamp: "2026-07-31T14:59:05Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-31T14:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T14:59:05Z" name: isvc-xgboost-graph-predictor.18c767d5f9481af5 namespace: kserve-ci-e2e-test resourceVersion: "15182" uid: 7ea33673-685a-4e69-aaee-f3cd5d110740 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T14:59:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: c8361f54-bbd1-46a7-bb79-65c11dc2ebd2 kind: Event lastTimestamp: "2026-07-31T15:00:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T14:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:21Z" name: isvc-xgboost-graph-predictor.18c767d977ab7009 namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 4098232b-c11e-4141-964f-67dc54c77e93 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T14:59:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: c8361f54-bbd1-46a7-bb79-65c11dc2ebd2 kind: Event lastTimestamp: "2026-07-31T15:00:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T14:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:21Z" name: isvc-xgboost-graph-predictor.18c767d9781794ad namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: f50583f5-54e7-48d5-a84b-dd7cd6373acf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T15:00:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: c8361f54-bbd1-46a7-bb79-65c11dc2ebd2 kind: Event lastTimestamp: "2026-07-31T15:01:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:21Z" name: isvc-xgboost-graph-predictor.18c767eaf2978cb5 namespace: kserve-ci-e2e-test resourceVersion: "16677" uid: 45dbc7fc-b389-4c2e-8849-d441ed4cd133 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T15:00:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15421" uid: c8361f54-bbd1-46a7-bb79-65c11dc2ebd2 kind: Event lastTimestamp: "2026-07-31T15:01:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:21Z" name: isvc-xgboost-graph-predictor.18c767eaf44afb9e namespace: kserve-ci-e2e-test resourceVersion: "16678" uid: 89153ac4-c246-46fc-b265-f90426b6d7db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-31T14:59:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15155" uid: 78f745b5-93ad-4887-90f3-9a9d298ae7ff kind: Event lastTimestamp: "2026-07-31T15:00:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T14:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:48Z" name: isvc-xgboost-graph.18c767d5f92ccc54 namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: 24315b32-230d-4a22-8366-3d157e51f704 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16386" uid: 78f745b5-93ad-4887-90f3-9a9d298ae7ff kind: Event lastTimestamp: "2026-07-31T15:00:48Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-31T15:00:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:48Z" name: isvc-xgboost-graph.18c767edc0f21640 namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: 5e08c699-0b4d-464e-b0cb-66eb8d5b486d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:01:25.692780Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-8c9487985-fvxx2 namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: 824e8e17-9d3b-42be-8ea8-b5f2ced26d8b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-8c9487985-fvxx2 to ip-10-0-136-197.ec2.internal metadata: creationTimestamp: "2026-07-31T15:01:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:01:25Z" name: model-chainer-8c9487985-fvxx2.18c767f680988446 namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 6818b6ef-921a-4d28-bbca-722c1061c051 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:26Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-8c9487985-fvxx2 namespace: kserve-ci-e2e-test resourceVersion: "16734" uid: 824e8e17-9d3b-42be-8ea8-b5f2ced26d8b kind: Event lastTimestamp: "2026-07-31T15:01:26Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:01:26Z" name: model-chainer-8c9487985-fvxx2.18c767f69a48978c namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: 87d31e2e-c891-4bb2-b828-b2ac3c4a7495 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8c9487985-fvxx2 namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 824e8e17-9d3b-42be-8ea8-b5f2ced26d8b kind: Event lastTimestamp: "2026-07-31T15:01:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:45f424b4a00ee86756776b91e414e15d8d34d9dfbb27b1d7b22dfeb283636eda" already present on machine metadata: creationTimestamp: "2026-07-31T15:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:26Z" name: model-chainer-8c9487985-fvxx2.18c767f69bac2cc5 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: 8b4d8ea1-54c3-491c-8035-b1ad4f103232 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8c9487985-fvxx2 namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 824e8e17-9d3b-42be-8ea8-b5f2ced26d8b kind: Event lastTimestamp: "2026-07-31T15:01:26Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-31T15:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:26Z" name: model-chainer-8c9487985-fvxx2.18c767f69fff5c7d namespace: kserve-ci-e2e-test resourceVersion: "16750" uid: 7ec0fd44-15a8-45f1-85a8-3f5cfdb601de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8c9487985-fvxx2 namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 824e8e17-9d3b-42be-8ea8-b5f2ced26d8b kind: Event lastTimestamp: "2026-07-31T15:01:26Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-31T15:01:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:26Z" name: model-chainer-8c9487985-fvxx2.18c767f6a0768b46 namespace: kserve-ci-e2e-test resourceVersion: "16751" uid: 2530cce0-c93d-4f0c-9db0-be891f94ff8f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8c9487985-fvxx2 namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 824e8e17-9d3b-42be-8ea8-b5f2ced26d8b kind: Event lastTimestamp: "2026-07-31T15:01:35Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-31T15:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:35Z" name: model-chainer-8c9487985-fvxx2.18c767f8da8de342 namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: bdc50787-2d6f-40c0-8a82-57cc0fdb4777 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:01:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-8c9487985-fvxx2 namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: 824e8e17-9d3b-42be-8ea8-b5f2ced26d8b kind: Event lastTimestamp: "2026-07-31T15:02:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-31T15:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:02:02Z" name: model-chainer-8c9487985-fvxx2.18c767f94988180f namespace: kserve-ci-e2e-test resourceVersion: "17270" uid: 8a57d39a-325d-41e8-b70a-d6193b5b1a15 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-8c9487985 namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 9b9f8bc1-885d-4562-9865-57438bb702dc kind: Event lastTimestamp: "2026-07-31T15:01:25Z" message: 'Created pod: model-chainer-8c9487985-fvxx2' metadata: creationTimestamp: "2026-07-31T15:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:25Z" name: model-chainer-8c9487985.18c767f68011ebeb namespace: kserve-ci-e2e-test resourceVersion: "16728" uid: 59d1f270-ac5c-4414-8ea8-ff41a7e8e55b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16721" uid: 81ca6469-6acd-4f14-8b73-282ddbd18239 kind: Event lastTimestamp: "2026-07-31T15:01:25Z" message: Scaled up replica set model-chainer-8c9487985 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:25Z" name: model-chainer.18c767f67edd2731 namespace: kserve-ci-e2e-test resourceVersion: "16724" uid: 68036edd-5e91-446a-907f-6f7d9b26751e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: 9f615f6d-5d86-4ab1-a7f3-752184a54293 kind: Event lastTimestamp: "2026-07-31T15:01:25Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-31T15:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:01:25Z" name: model-chainer.18c767f684e6ca92 namespace: kserve-ci-e2e-test resourceVersion: "16741" uid: 53bf5473-651b-4d78-8f12-fb591d04adc8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:13:15.369437Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7b982-6788699d75-lth46 namespace: kserve-ci-e2e-test resourceVersion: "22909" uid: 32ea8c9d-a07d-49f0-87eb-2986ea2dea94 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-7b982-6788699d75-lth46 to ip-10-0-136-197.ec2.internal metadata: creationTimestamp: "2026-07-31T15:13:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:13:15Z" name: sequence-graph-7b982-6788699d75-lth46.18c7689bbc9ef21e namespace: kserve-ci-e2e-test resourceVersion: "22914" uid: 77bd7c75-940e-4049-b995-b5dab7db933f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:15Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7b982-6788699d75-lth46 namespace: kserve-ci-e2e-test resourceVersion: "22911" uid: 32ea8c9d-a07d-49f0-87eb-2986ea2dea94 kind: Event lastTimestamp: "2026-07-31T15:13:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-7b982-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T15:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:13:15Z" name: sequence-graph-7b982-6788699d75-lth46.18c7689bc4888321 namespace: kserve-ci-e2e-test resourceVersion: "22927" uid: 9163a57d-ab81-4223-87b4-ea0b9c9d13c6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:16Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7b982-6788699d75-lth46 namespace: kserve-ci-e2e-test resourceVersion: "22917" uid: 32ea8c9d-a07d-49f0-87eb-2986ea2dea94 kind: Event lastTimestamp: "2026-07-31T15:13:16Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:13:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:13:16Z" name: sequence-graph-7b982-6788699d75-lth46.18c7689bf9fa462f namespace: kserve-ci-e2e-test resourceVersion: "22934" uid: b53ed9c7-ea96-47ef-aed8-099417a298ef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7b982} kind: Pod name: sequence-graph-7b982-6788699d75-lth46 namespace: kserve-ci-e2e-test resourceVersion: "22911" uid: 32ea8c9d-a07d-49f0-87eb-2986ea2dea94 kind: Event lastTimestamp: "2026-07-31T15:13:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:45f424b4a00ee86756776b91e414e15d8d34d9dfbb27b1d7b22dfeb283636eda" already present on machine metadata: creationTimestamp: "2026-07-31T15:13:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:13:16Z" name: sequence-graph-7b982-6788699d75-lth46.18c7689bfb47c438 namespace: kserve-ci-e2e-test resourceVersion: "22936" uid: 2fd9a858-affe-43f9-9018-cbdf1196aadc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7b982} kind: Pod name: sequence-graph-7b982-6788699d75-lth46 namespace: kserve-ci-e2e-test resourceVersion: "22911" uid: 32ea8c9d-a07d-49f0-87eb-2986ea2dea94 kind: Event lastTimestamp: "2026-07-31T15:13:16Z" message: 'Created container: sequence-graph-7b982' metadata: creationTimestamp: "2026-07-31T15:13:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:13:16Z" name: sequence-graph-7b982-6788699d75-lth46.18c7689bfe4e4707 namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: 16fb45ec-7b4a-4c30-aadd-13676dcd0db8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7b982} kind: Pod name: sequence-graph-7b982-6788699d75-lth46 namespace: kserve-ci-e2e-test resourceVersion: "22911" uid: 32ea8c9d-a07d-49f0-87eb-2986ea2dea94 kind: Event lastTimestamp: "2026-07-31T15:13:16Z" message: Started container sequence-graph-7b982 metadata: creationTimestamp: "2026-07-31T15:13:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:13:16Z" name: sequence-graph-7b982-6788699d75-lth46.18c7689bfed585a6 namespace: kserve-ci-e2e-test resourceVersion: "22938" uid: 5513bfec-6674-4bfb-be19-7bfe10393f9c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7b982} kind: Pod name: sequence-graph-7b982-6788699d75-lth46 namespace: kserve-ci-e2e-test resourceVersion: "22911" uid: 32ea8c9d-a07d-49f0-87eb-2986ea2dea94 kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Stopping container sequence-graph-7b982 metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:30Z" name: sequence-graph-7b982-6788699d75-lth46.18c7690ee9452fc8 namespace: kserve-ci-e2e-test resourceVersion: "26862" uid: c221f6fc-289a-4a5a-9bc8-e7129b50060c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7b982} kind: Pod name: sequence-graph-7b982-6788699d75-lth46 namespace: kserve-ci-e2e-test resourceVersion: "22911" uid: 32ea8c9d-a07d-49f0-87eb-2986ea2dea94 kind: Event lastTimestamp: "2026-07-31T15:21:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:57Z" name: sequence-graph-7b982-6788699d75-lth46.18c7690f952000f7 namespace: kserve-ci-e2e-test resourceVersion: "27376" uid: c66782b0-10e7-400b-a404-864c8f827fab reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-7b982-6788699d75 namespace: kserve-ci-e2e-test resourceVersion: "22905" uid: 55efb9b1-b2f3-4844-94bc-67fdc90ab9fa kind: Event lastTimestamp: "2026-07-31T15:13:15Z" message: 'Created pod: sequence-graph-7b982-6788699d75-lth46' metadata: creationTimestamp: "2026-07-31T15:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:13:15Z" name: sequence-graph-7b982-6788699d75.18c7689bbc2b7cdd namespace: kserve-ci-e2e-test resourceVersion: "22910" uid: 11d45c66-890d-4c0a-a64d-4b40df5a6765 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-7b982 namespace: kserve-ci-e2e-test resourceVersion: "22902" uid: 67c701a4-524c-4159-ac34-73b7c3858ab2 kind: Event lastTimestamp: "2026-07-31T15:13:15Z" message: Scaled up replica set sequence-graph-7b982-6788699d75 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:13:15Z" name: sequence-graph-7b982.18c7689bbb323a47 namespace: kserve-ci-e2e-test resourceVersion: "22907" uid: 6353c672-c6b4-45e4-8add-a3b386b7b8e8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-7b982 namespace: kserve-ci-e2e-test resourceVersion: "22920" uid: 8c9cc22b-ef60-4f8a-a163-605a7f717d56 kind: Event lastTimestamp: "2026-07-31T15:13:15Z" message: InferenceGraph [sequence-graph-7b982] is Ready metadata: creationTimestamp: "2026-07-31T15:13:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:13:15Z" name: sequence-graph-7b982.18c7689bc1100420 namespace: kserve-ci-e2e-test resourceVersion: "22923" uid: 8eb59c2f-c34e-4abf-b919-6234bedb5c18 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7b982 namespace: kserve-ci-e2e-test resourceVersion: "22903" uid: f836e559-f6c4-48e9-8455-66dbab92a754 kind: Event lastTimestamp: "2026-07-31T15:13:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:13:30Z" name: sequence-graph-7b982.18c7689f398f5642 namespace: kserve-ci-e2e-test resourceVersion: "23040" uid: b0f3c31e-f953-410b-a11a-aceae6f1ddbd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7b982 namespace: kserve-ci-e2e-test resourceVersion: "22903" uid: f836e559-f6c4-48e9-8455-66dbab92a754 kind: Event lastTimestamp: "2026-07-31T15:13:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:13:30Z" name: sequence-graph-7b982.18c7689f39efce8c namespace: kserve-ci-e2e-test resourceVersion: "23042" uid: a1509198-e2e3-4209-bc4d-411c03d9caba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7b982 namespace: kserve-ci-e2e-test resourceVersion: "23041" uid: f836e559-f6c4-48e9-8455-66dbab92a754 kind: Event lastTimestamp: "2026-07-31T15:13:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:13:45Z" name: sequence-graph-7b982.18c768a2b8ada864 namespace: kserve-ci-e2e-test resourceVersion: "23152" uid: 93c34cd4-6b39-4384-b128-788ae70fb489 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-7b982 namespace: kserve-ci-e2e-test resourceVersion: "23041" uid: f836e559-f6c4-48e9-8455-66dbab92a754 kind: Event lastTimestamp: "2026-07-31T15:13:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:13:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:13:45Z" name: sequence-graph-7b982.18c768a2b91e62a7 namespace: kserve-ci-e2e-test resourceVersion: "23154" uid: f53838e3-33f7-423e-bb05-38e448194fb7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:12:00.956669Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a1442-9fd45fd4d-bpdzh namespace: kserve-ci-e2e-test resourceVersion: "22084" uid: f465af3a-43ec-4f14-a323-78de0bc11d88 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a1442-9fd45fd4d-bpdzh to ip-10-0-136-197.ec2.internal metadata: creationTimestamp: "2026-07-31T15:12:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:12:00Z" name: sequence-graph-a1442-9fd45fd4d-bpdzh.18c7688a69463627 namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: ba26ead1-ebdc-4d27-b5d2-d6cd5573ff55 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:01Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a1442-9fd45fd4d-bpdzh namespace: kserve-ci-e2e-test resourceVersion: "22094" uid: f465af3a-43ec-4f14-a323-78de0bc11d88 kind: Event lastTimestamp: "2026-07-31T15:12:01Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:12:01Z" name: sequence-graph-a1442-9fd45fd4d-bpdzh.18c7688a8305c760 namespace: kserve-ci-e2e-test resourceVersion: "22103" uid: 99513146-39a4-496a-a91f-794ca59a5119 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a1442} kind: Pod name: sequence-graph-a1442-9fd45fd4d-bpdzh namespace: kserve-ci-e2e-test resourceVersion: "22086" uid: f465af3a-43ec-4f14-a323-78de0bc11d88 kind: Event lastTimestamp: "2026-07-31T15:12:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:45f424b4a00ee86756776b91e414e15d8d34d9dfbb27b1d7b22dfeb283636eda" already present on machine metadata: creationTimestamp: "2026-07-31T15:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:01Z" name: sequence-graph-a1442-9fd45fd4d-bpdzh.18c7688a847b4e82 namespace: kserve-ci-e2e-test resourceVersion: "22105" uid: 1c9ebf55-8cdc-461c-9e10-232a5f28c5bc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a1442} kind: Pod name: sequence-graph-a1442-9fd45fd4d-bpdzh namespace: kserve-ci-e2e-test resourceVersion: "22086" uid: f465af3a-43ec-4f14-a323-78de0bc11d88 kind: Event lastTimestamp: "2026-07-31T15:12:01Z" message: 'Created container: sequence-graph-a1442' metadata: creationTimestamp: "2026-07-31T15:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:01Z" name: sequence-graph-a1442-9fd45fd4d-bpdzh.18c7688a878b4550 namespace: kserve-ci-e2e-test resourceVersion: "22107" uid: 8f4f8317-a431-459c-ba96-a58e00360ce5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a1442} kind: Pod name: sequence-graph-a1442-9fd45fd4d-bpdzh namespace: kserve-ci-e2e-test resourceVersion: "22086" uid: f465af3a-43ec-4f14-a323-78de0bc11d88 kind: Event lastTimestamp: "2026-07-31T15:12:01Z" message: Started container sequence-graph-a1442 metadata: creationTimestamp: "2026-07-31T15:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:01Z" name: sequence-graph-a1442-9fd45fd4d-bpdzh.18c7688a87fd6e60 namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: b3d299fc-d356-494d-92c6-390641f95d60 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a1442} kind: Pod name: sequence-graph-a1442-9fd45fd4d-bpdzh namespace: kserve-ci-e2e-test resourceVersion: "22086" uid: f465af3a-43ec-4f14-a323-78de0bc11d88 kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: Stopping container sequence-graph-a1442 metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:15Z" name: sequence-graph-a1442-9fd45fd4d-bpdzh.18c7688db2019ba8 namespace: kserve-ci-e2e-test resourceVersion: "22280" uid: 5adb4e9f-a0de-4bd8-94b5-1c09ca90ad4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:12:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a1442} kind: Pod name: sequence-graph-a1442-9fd45fd4d-bpdzh namespace: kserve-ci-e2e-test resourceVersion: "22086" uid: f465af3a-43ec-4f14-a323-78de0bc11d88 kind: Event lastTimestamp: "2026-07-31T15:12:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-31T15:12:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:42Z" name: sequence-graph-a1442-9fd45fd4d-bpdzh.18c7688e479e16d7 namespace: kserve-ci-e2e-test resourceVersion: "22628" uid: c6ba95dc-681a-4b72-a23e-fe0bde3ec591 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a1442-9fd45fd4d namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: ec35a98b-98ab-4235-939d-fa3b8fe06110 kind: Event lastTimestamp: "2026-07-31T15:12:00Z" message: 'Created pod: sequence-graph-a1442-9fd45fd4d-bpdzh' metadata: creationTimestamp: "2026-07-31T15:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:00Z" name: sequence-graph-a1442-9fd45fd4d.18c7688a68a34b36 namespace: kserve-ci-e2e-test resourceVersion: "22085" uid: ef9825a8-1de4-4a84-ba61-db30c5517b7e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a1442 namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: e5fcd828-78f2-4be7-98b3-45cc83c3205c kind: Event lastTimestamp: "2026-07-31T15:12:00Z" message: Scaled up replica set sequence-graph-a1442-9fd45fd4d from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:00Z" name: sequence-graph-a1442.18c7688a67c91558 namespace: kserve-ci-e2e-test resourceVersion: "22082" uid: 3ba85e14-3323-4d34-8c7e-9da2f887b253 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a1442 namespace: kserve-ci-e2e-test resourceVersion: "22096" uid: 97cc22b4-f132-4ccc-9250-06ee4587c37a kind: Event lastTimestamp: "2026-07-31T15:12:01Z" message: InferenceGraph [sequence-graph-a1442] is Ready metadata: creationTimestamp: "2026-07-31T15:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:12:01Z" name: sequence-graph-a1442.18c7688a6d694837 namespace: kserve-ci-e2e-test resourceVersion: "22098" uid: fa4929a0-bf84-427f-b6b8-80dec2c6dc24 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:02:36.008726Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a9565-55fcbd5d96-wt9cc namespace: kserve-ci-e2e-test resourceVersion: "17543" uid: 5fbab7b4-19bf-473a-8da2-9e56c2dfaa5a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a9565-55fcbd5d96-wt9cc to ip-10-0-136-197.ec2.internal metadata: creationTimestamp: "2026-07-31T15:02:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:02:36Z" name: sequence-graph-a9565-55fcbd5d96-wt9cc.18c76806dfc0c146 namespace: kserve-ci-e2e-test resourceVersion: "17548" uid: 44aae93f-781f-4804-aa65-acb1d72dfedc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:02:36Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a9565-55fcbd5d96-wt9cc namespace: kserve-ci-e2e-test resourceVersion: "17546" uid: 5fbab7b4-19bf-473a-8da2-9e56c2dfaa5a kind: Event lastTimestamp: "2026-07-31T15:02:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-a9565-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T15:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:02:36Z" name: sequence-graph-a9565-55fcbd5d96-wt9cc.18c76806e74145ab namespace: kserve-ci-e2e-test resourceVersion: "17567" uid: bdcf738f-1b4b-4c68-bd9f-16a4ac400568 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:02:37Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a9565-55fcbd5d96-wt9cc namespace: kserve-ci-e2e-test resourceVersion: "17555" uid: 5fbab7b4-19bf-473a-8da2-9e56c2dfaa5a kind: Event lastTimestamp: "2026-07-31T15:02:37Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:02:37Z" name: sequence-graph-a9565-55fcbd5d96-wt9cc.18c7680752b684f1 namespace: kserve-ci-e2e-test resourceVersion: "17580" uid: 1c26c466-8ad1-4305-80f9-a271351df8fc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:02:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a9565} kind: Pod name: sequence-graph-a9565-55fcbd5d96-wt9cc namespace: kserve-ci-e2e-test resourceVersion: "17546" uid: 5fbab7b4-19bf-473a-8da2-9e56c2dfaa5a kind: Event lastTimestamp: "2026-07-31T15:02:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:45f424b4a00ee86756776b91e414e15d8d34d9dfbb27b1d7b22dfeb283636eda" already present on machine metadata: creationTimestamp: "2026-07-31T15:02:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:02:37Z" name: sequence-graph-a9565-55fcbd5d96-wt9cc.18c76807540b7b05 namespace: kserve-ci-e2e-test resourceVersion: "17582" uid: c3b27c1f-95d0-4a2b-a1f2-c056329e0160 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a9565} kind: Pod name: sequence-graph-a9565-55fcbd5d96-wt9cc namespace: kserve-ci-e2e-test resourceVersion: "17546" uid: 5fbab7b4-19bf-473a-8da2-9e56c2dfaa5a kind: Event lastTimestamp: "2026-07-31T15:02:38Z" message: 'Created container: sequence-graph-a9565' metadata: creationTimestamp: "2026-07-31T15:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:02:38Z" name: sequence-graph-a9565-55fcbd5d96-wt9cc.18c7680757423704 namespace: kserve-ci-e2e-test resourceVersion: "17583" uid: 5a89f2fd-eec5-4a9d-b325-5f99df4258a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:02:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a9565} kind: Pod name: sequence-graph-a9565-55fcbd5d96-wt9cc namespace: kserve-ci-e2e-test resourceVersion: "17546" uid: 5fbab7b4-19bf-473a-8da2-9e56c2dfaa5a kind: Event lastTimestamp: "2026-07-31T15:02:38Z" message: Started container sequence-graph-a9565 metadata: creationTimestamp: "2026-07-31T15:02:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:02:38Z" name: sequence-graph-a9565-55fcbd5d96-wt9cc.18c7680757b2fac4 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: cf80dcf9-7069-4bb0-93b1-334df901014d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a9565} kind: Pod name: sequence-graph-a9565-55fcbd5d96-wt9cc namespace: kserve-ci-e2e-test resourceVersion: "17546" uid: 5fbab7b4-19bf-473a-8da2-9e56c2dfaa5a kind: Event lastTimestamp: "2026-07-31T15:10:50Z" message: Stopping container sequence-graph-a9565 metadata: creationTimestamp: "2026-07-31T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:50Z" name: sequence-graph-a9565-55fcbd5d96-wt9cc.18c7687a0f504eb1 namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: f2476223-6632-4c63-a369-875fd2f31d4a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:10:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a9565} kind: Pod name: sequence-graph-a9565-55fcbd5d96-wt9cc namespace: kserve-ci-e2e-test resourceVersion: "17546" uid: 5fbab7b4-19bf-473a-8da2-9e56c2dfaa5a kind: Event lastTimestamp: "2026-07-31T15:11:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-31T15:10:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:20Z" name: sequence-graph-a9565-55fcbd5d96-wt9cc.18c7687b0ea927c5 namespace: kserve-ci-e2e-test resourceVersion: "21718" uid: 3bd475ca-e7a5-43d1-af0a-192435341ee6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:02:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a9565-55fcbd5d96 namespace: kserve-ci-e2e-test resourceVersion: "17540" uid: 6ce7f939-3c7d-418c-ab8b-292fd6c48b58 kind: Event lastTimestamp: "2026-07-31T15:02:36Z" message: 'Created pod: sequence-graph-a9565-55fcbd5d96-wt9cc' metadata: creationTimestamp: "2026-07-31T15:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:02:36Z" name: sequence-graph-a9565-55fcbd5d96.18c76806df4ec11c namespace: kserve-ci-e2e-test resourceVersion: "17545" uid: 7a709713-9b8c-4718-a5b5-38e939d9813c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:02:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a9565 namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 81bc6167-ea57-4ffe-8d45-e515699fe6f5 kind: Event lastTimestamp: "2026-07-31T15:02:35Z" message: Scaled up replica set sequence-graph-a9565-55fcbd5d96 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:02:35Z" name: sequence-graph-a9565.18c76806de4dcb61 namespace: kserve-ci-e2e-test resourceVersion: "17541" uid: 5e3ca0ea-9f1f-4f1b-bccb-a64edeccec3e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:02:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a9565 namespace: kserve-ci-e2e-test resourceVersion: "17557" uid: 141900ed-6f3b-4a81-a144-d6faccde49eb kind: Event lastTimestamp: "2026-07-31T15:02:36Z" message: InferenceGraph [sequence-graph-a9565] is Ready metadata: creationTimestamp: "2026-07-31T15:02:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:02:36Z" name: sequence-graph-a9565.18c76806e3e2e26c namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: d8f94e6e-f515-4364-b9a0-15e37f7f7364 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:02:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a9565 namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: 4c5a2b5b-8f09-4697-9016-4df9ec49a3a1 kind: Event lastTimestamp: "2026-07-31T15:02:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:02:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:02:50Z" name: sequence-graph-a9565.18c7680a5cc1da1c namespace: kserve-ci-e2e-test resourceVersion: "17683" uid: 49e6a3fa-20dd-4745-8579-b393dd0d281c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:02:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a9565 namespace: kserve-ci-e2e-test resourceVersion: "17539" uid: 4c5a2b5b-8f09-4697-9016-4df9ec49a3a1 kind: Event lastTimestamp: "2026-07-31T15:02:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:02:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:02:50Z" name: sequence-graph-a9565.18c7680a5d328d7e namespace: kserve-ci-e2e-test resourceVersion: "17685" uid: 1e2b9697-9974-44c0-b791-548d0f0baf08 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:03:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a9565 namespace: kserve-ci-e2e-test resourceVersion: "17684" uid: 4c5a2b5b-8f09-4697-9016-4df9ec49a3a1 kind: Event lastTimestamp: "2026-07-31T15:03:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:03:06Z" name: sequence-graph-a9565.18c7680ddc3aee87 namespace: kserve-ci-e2e-test resourceVersion: "17789" uid: 170e41b2-0b76-4318-bfe6-5d56968072e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:03:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a9565 namespace: kserve-ci-e2e-test resourceVersion: "17684" uid: 4c5a2b5b-8f09-4697-9016-4df9ec49a3a1 kind: Event lastTimestamp: "2026-07-31T15:03:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:03:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:03:06Z" name: sequence-graph-a9565.18c7680ddc833094 namespace: kserve-ci-e2e-test resourceVersion: "17790" uid: 8e2e6dc8-4200-4b52-bfda-b0fc7704542d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:21:20.310465Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e31d5-564c58485c-xhmft namespace: kserve-ci-e2e-test resourceVersion: "26744" uid: c018439e-ceb9-43e5-9f50-48e4b0a195e6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-e31d5-564c58485c-xhmft to ip-10-0-136-197.ec2.internal metadata: creationTimestamp: "2026-07-31T15:21:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:21:20Z" name: splitter-graph-e31d5-564c58485c-xhmft.18c7690ca55bc3c7 namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: f7aad48c-9443-41dd-a411-a4d0cb2d41e2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:20Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e31d5-564c58485c-xhmft namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: c018439e-ceb9-43e5-9f50-48e4b0a195e6 kind: Event lastTimestamp: "2026-07-31T15:21:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-e31d5-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T15:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:20Z" name: splitter-graph-e31d5-564c58485c-xhmft.18c7690cb424c069 namespace: kserve-ci-e2e-test resourceVersion: "26764" uid: f530470e-fb9c-43fc-b8a8-ccab776eb633 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:21Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e31d5-564c58485c-xhmft namespace: kserve-ci-e2e-test resourceVersion: "26756" uid: c018439e-ceb9-43e5-9f50-48e4b0a195e6 kind: Event lastTimestamp: "2026-07-31T15:21:21Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:21:21Z" name: splitter-graph-e31d5-564c58485c-xhmft.18c7690ce2ad7e86 namespace: kserve-ci-e2e-test resourceVersion: "26772" uid: 03fee546-61fa-4b98-8b7b-2c7121ee6d9d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e31d5} kind: Pod name: splitter-graph-e31d5-564c58485c-xhmft namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: c018439e-ceb9-43e5-9f50-48e4b0a195e6 kind: Event lastTimestamp: "2026-07-31T15:21:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:45f424b4a00ee86756776b91e414e15d8d34d9dfbb27b1d7b22dfeb283636eda" already present on machine metadata: creationTimestamp: "2026-07-31T15:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:21Z" name: splitter-graph-e31d5-564c58485c-xhmft.18c7690ce4374552 namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: 220bc7ae-0048-4329-b1ed-078403f703e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e31d5} kind: Pod name: splitter-graph-e31d5-564c58485c-xhmft namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: c018439e-ceb9-43e5-9f50-48e4b0a195e6 kind: Event lastTimestamp: "2026-07-31T15:21:21Z" message: 'Created container: splitter-graph-e31d5' metadata: creationTimestamp: "2026-07-31T15:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:21Z" name: splitter-graph-e31d5-564c58485c-xhmft.18c7690ce7226d82 namespace: kserve-ci-e2e-test resourceVersion: "26775" uid: 28abe80e-6c53-44c5-b66e-84afee8d35ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e31d5} kind: Pod name: splitter-graph-e31d5-564c58485c-xhmft namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: c018439e-ceb9-43e5-9f50-48e4b0a195e6 kind: Event lastTimestamp: "2026-07-31T15:21:21Z" message: Started container splitter-graph-e31d5 metadata: creationTimestamp: "2026-07-31T15:21:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:21Z" name: splitter-graph-e31d5-564c58485c-xhmft.18c7690ce7a488cd namespace: kserve-ci-e2e-test resourceVersion: "26776" uid: ae5b46f3-acc7-4ca1-b0e9-05da581b3961 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e31d5} kind: Pod name: splitter-graph-e31d5-564c58485c-xhmft namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: c018439e-ceb9-43e5-9f50-48e4b0a195e6 kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Stopping container splitter-graph-e31d5 metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:30Z" name: splitter-graph-e31d5-564c58485c-xhmft.18c7690f003e0f12 namespace: kserve-ci-e2e-test resourceVersion: "26951" uid: 95d6e9cd-e358-4f5b-9641-cb52fe47f857 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:21:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e31d5} kind: Pod name: splitter-graph-e31d5-564c58485c-xhmft namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: c018439e-ceb9-43e5-9f50-48e4b0a195e6 kind: Event lastTimestamp: "2026-07-31T15:21:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-31T15:21:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:58Z" name: splitter-graph-e31d5-564c58485c-xhmft.18c7690fa28dd6ce namespace: kserve-ci-e2e-test resourceVersion: "27377" uid: 627030f5-bfcc-40ae-af37-4d6fc55906f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-e31d5-564c58485c namespace: kserve-ci-e2e-test resourceVersion: "26741" uid: 118aee9e-be23-40c2-9809-fc66892bd437 kind: Event lastTimestamp: "2026-07-31T15:21:20Z" message: 'Created pod: splitter-graph-e31d5-564c58485c-xhmft' metadata: creationTimestamp: "2026-07-31T15:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:20Z" name: splitter-graph-e31d5-564c58485c.18c7690ca49524b7 namespace: kserve-ci-e2e-test resourceVersion: "26748" uid: 90460d13-9768-4195-bd4a-90ea9dc2760a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-e31d5 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: eaa358fe-1ac9-4882-a100-ec5b99110f1b kind: Event lastTimestamp: "2026-07-31T15:21:20Z" message: Scaled up replica set splitter-graph-e31d5-564c58485c from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:20Z" name: splitter-graph-e31d5.18c7690ca2b3efaa namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: db6cc64f-81d2-4779-aada-09372982e704 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-e31d5 namespace: kserve-ci-e2e-test resourceVersion: "26758" uid: 5dc49fb8-8b4d-4893-b2db-b8a831baf6be kind: Event lastTimestamp: "2026-07-31T15:21:20Z" message: InferenceGraph [splitter-graph-e31d5] is Ready metadata: creationTimestamp: "2026-07-31T15:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:21:20Z" name: splitter-graph-e31d5.18c7690ca985756e namespace: kserve-ci-e2e-test resourceVersion: "26760" uid: 3741343d-911f-4c79-8bb7-0ed355d9b1a5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:22:40.704864Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-faf8d-7794748f4c-g24jk namespace: kserve-ci-e2e-test resourceVersion: "27795" uid: adaf092f-278d-4453-88df-4af746f15dee kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-faf8d-7794748f4c-g24jk to ip-10-0-136-197.ec2.internal metadata: creationTimestamp: "2026-07-31T15:22:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:22:40Z" name: splitter-graph-faf8d-7794748f4c-g24jk.18c7691f5d3d03b5 namespace: kserve-ci-e2e-test resourceVersion: "27801" uid: 67880fa6-c3e8-46c3-ba9c-1ac8ae5ad42c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:41Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-faf8d-7794748f4c-g24jk namespace: kserve-ci-e2e-test resourceVersion: "27805" uid: adaf092f-278d-4453-88df-4af746f15dee kind: Event lastTimestamp: "2026-07-31T15:22:41Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:22:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:22:41Z" name: splitter-graph-faf8d-7794748f4c-g24jk.18c7691f76c7cbea namespace: kserve-ci-e2e-test resourceVersion: "27816" uid: 0b8523d1-766e-4b4f-8394-59a951338a3f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-faf8d} kind: Pod name: splitter-graph-faf8d-7794748f4c-g24jk namespace: kserve-ci-e2e-test resourceVersion: "27799" uid: adaf092f-278d-4453-88df-4af746f15dee kind: Event lastTimestamp: "2026-07-31T15:22:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:45f424b4a00ee86756776b91e414e15d8d34d9dfbb27b1d7b22dfeb283636eda" already present on machine metadata: creationTimestamp: "2026-07-31T15:22:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:22:41Z" name: splitter-graph-faf8d-7794748f4c-g24jk.18c7691f7863129a namespace: kserve-ci-e2e-test resourceVersion: "27819" uid: bc8a1195-b3e0-42a2-8c74-2638dca987c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-faf8d} kind: Pod name: splitter-graph-faf8d-7794748f4c-g24jk namespace: kserve-ci-e2e-test resourceVersion: "27799" uid: adaf092f-278d-4453-88df-4af746f15dee kind: Event lastTimestamp: "2026-07-31T15:22:41Z" message: 'Created container: splitter-graph-faf8d' metadata: creationTimestamp: "2026-07-31T15:22:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:22:41Z" name: splitter-graph-faf8d-7794748f4c-g24jk.18c7691f7c1c988f namespace: kserve-ci-e2e-test resourceVersion: "27820" uid: 3e460c8b-3a31-4040-a9e8-51f66bb195b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-faf8d} kind: Pod name: splitter-graph-faf8d-7794748f4c-g24jk namespace: kserve-ci-e2e-test resourceVersion: "27799" uid: adaf092f-278d-4453-88df-4af746f15dee kind: Event lastTimestamp: "2026-07-31T15:22:41Z" message: Started container splitter-graph-faf8d metadata: creationTimestamp: "2026-07-31T15:22:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:22:41Z" name: splitter-graph-faf8d-7794748f4c-g24jk.18c7691f7cb2c730 namespace: kserve-ci-e2e-test resourceVersion: "27821" uid: c0c9f4d5-845b-48fe-a1e0-8f874f1bdf2a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:30:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-faf8d} kind: Pod name: splitter-graph-faf8d-7794748f4c-g24jk namespace: kserve-ci-e2e-test resourceVersion: "27799" uid: adaf092f-278d-4453-88df-4af746f15dee kind: Event lastTimestamp: "2026-07-31T15:30:55Z" message: Stopping container splitter-graph-faf8d metadata: creationTimestamp: "2026-07-31T15:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:30:55Z" name: splitter-graph-faf8d-7794748f4c-g24jk.18c7699287aec672 namespace: kserve-ci-e2e-test resourceVersion: "31253" uid: fe4f2668-c12e-4349-a55b-2ed2ef20f26c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:30:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-faf8d} kind: Pod name: splitter-graph-faf8d-7794748f4c-g24jk namespace: kserve-ci-e2e-test resourceVersion: "27799" uid: adaf092f-278d-4453-88df-4af746f15dee kind: Event lastTimestamp: "2026-07-31T15:31:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-31T15:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:31:22Z" name: splitter-graph-faf8d-7794748f4c-g24jk.18c7699308fc3e19 namespace: kserve-ci-e2e-test resourceVersion: "31486" uid: cea9cc35-d591-4eb2-82f6-438284c1a3cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-faf8d-7794748f4c namespace: kserve-ci-e2e-test resourceVersion: "27792" uid: e1cd2759-ed1b-4ef6-ab68-acba6256d058 kind: Event lastTimestamp: "2026-07-31T15:22:40Z" message: 'Created pod: splitter-graph-faf8d-7794748f4c-g24jk' metadata: creationTimestamp: "2026-07-31T15:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:40Z" name: splitter-graph-faf8d-7794748f4c.18c7691f5c49312b namespace: kserve-ci-e2e-test resourceVersion: "27796" uid: 150c4de9-4291-4301-8aca-967fb978f110 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-faf8d namespace: kserve-ci-e2e-test resourceVersion: "27790" uid: 714fa665-bdb6-4636-bedb-587502a4b8c9 kind: Event lastTimestamp: "2026-07-31T15:22:40Z" message: Scaled up replica set splitter-graph-faf8d-7794748f4c from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:40Z" name: splitter-graph-faf8d.18c7691f5b32f722 namespace: kserve-ci-e2e-test resourceVersion: "27794" uid: e6dff2cd-605a-48c7-964e-527c8d54faee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-faf8d namespace: kserve-ci-e2e-test resourceVersion: "27808" uid: bbc1ee24-126a-4955-83b4-116847657d0c kind: Event lastTimestamp: "2026-07-31T15:22:40Z" message: InferenceGraph [splitter-graph-faf8d] is Ready metadata: creationTimestamp: "2026-07-31T15:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:22:40Z" name: splitter-graph-faf8d.18c7691f61e23c0f namespace: kserve-ci-e2e-test resourceVersion: "27812" uid: e3d6c538-a601-40dd-9cbd-c604fb6e4f78 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:22:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-faf8d namespace: kserve-ci-e2e-test resourceVersion: "27791" uid: debf59f4-705b-4d1c-b050-d9ae3d34fd83 kind: Event lastTimestamp: "2026-07-31T15:23:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:22:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:23:10Z" name: splitter-graph-faf8d.18c76922d9aca1ae namespace: kserve-ci-e2e-test resourceVersion: "28049" uid: b03a619f-5b84-40e2-89e1-006f0bf5ff3f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:22:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-faf8d namespace: kserve-ci-e2e-test resourceVersion: "27791" uid: debf59f4-705b-4d1c-b050-d9ae3d34fd83 kind: Event lastTimestamp: "2026-07-31T15:23:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:22:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:23:10Z" name: splitter-graph-faf8d.18c76922da116998 namespace: kserve-ci-e2e-test resourceVersion: "28050" uid: f41e7856-b4f4-4d73-8861-1151f22d6811 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:11:05.337694Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 namespace: kserve-ci-e2e-test resourceVersion: "21511" uid: 5350a19f-b95b-4b88-afbb-ffa9574c5eae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:11:05Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296.18c7687d762005f6 namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 13ed7397-9e0a-4aba-b5a9-c7edad83d9ae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 5350a19f-b95b-4b88-afbb-ffa9574c5eae kind: Event lastTimestamp: "2026-07-31T15:11:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-06a6c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:05Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296.18c7687d7e5f7b00 namespace: kserve-ci-e2e-test resourceVersion: "21554" uid: 5becb0c2-8e78-4f3b-be8f-b5b97e92d75c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: 5350a19f-b95b-4b88-afbb-ffa9574c5eae kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:11:06Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296.18c7687db4778d30 namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: e024e609-7c5f-46c9-b8bc-cfce455ff550 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 5350a19f-b95b-4b88-afbb-ffa9574c5eae kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:06Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296.18c7687db77e58e7 namespace: kserve-ci-e2e-test resourceVersion: "21584" uid: 51a5467a-4d3a-4fec-9638-f410042038c9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 5350a19f-b95b-4b88-afbb-ffa9574c5eae kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:06Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296.18c7687dbb11d745 namespace: kserve-ci-e2e-test resourceVersion: "21587" uid: 1444d196-646e-43a6-80d8-fd2ccf5b9ca0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 5350a19f-b95b-4b88-afbb-ffa9574c5eae kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:06Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296.18c7687dbbc5078e namespace: kserve-ci-e2e-test resourceVersion: "21588" uid: 433e15a3-4d65-48a9-b9ed-861ef6afb216 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 5350a19f-b95b-4b88-afbb-ffa9574c5eae kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:06Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296.18c7687dbbe636b0 namespace: kserve-ci-e2e-test resourceVersion: "21589" uid: 14fea9ac-4b78-4ebe-aa3f-e8e08f0d29ab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 5350a19f-b95b-4b88-afbb-ffa9574c5eae kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:06Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296.18c7687dbed4d824 namespace: kserve-ci-e2e-test resourceVersion: "21591" uid: 95d5b621-c485-4eae-8bdf-aa3fab38543c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 5350a19f-b95b-4b88-afbb-ffa9574c5eae kind: Event lastTimestamp: "2026-07-31T15:11:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:06Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296.18c7687dbf52adb9 namespace: kserve-ci-e2e-test resourceVersion: "21593" uid: 2237d8da-c579-4334-814e-2701fddfb8ef reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T15:11:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 5350a19f-b95b-4b88-afbb-ffa9574c5eae kind: Event lastTimestamp: "2026-07-31T15:20:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:11:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:22Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296.18c7687dc36450ee namespace: kserve-ci-e2e-test resourceVersion: "26285" uid: 1cb44431-ad02-4d7c-bc41-ac3f074531c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 5350a19f-b95b-4b88-afbb-ffa9574c5eae kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296.18c768fea4c3121c namespace: kserve-ci-e2e-test resourceVersion: "26175" uid: 5182bb7a-7890-4ac7-b909-e1a4341e35ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 5350a19f-b95b-4b88-afbb-ffa9574c5eae kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296.18c768fea4c43c9c namespace: kserve-ci-e2e-test resourceVersion: "26177" uid: 2817ff01-65cf-477a-bfea-97f5b7757b32 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 5350a19f-b95b-4b88-afbb-ffa9574c5eae kind: Event lastTimestamp: "2026-07-31T15:20:22Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:20:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:22Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296.18c768ff37dfb89b namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: 96b5e184-78c1-4215-bbca-146e77b7a91e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-06a6c-predictor-69d6f4dc94 namespace: kserve-ci-e2e-test resourceVersion: "21495" uid: 954cc3db-d439-4868-ab3b-78f5d180bf62 kind: Event lastTimestamp: "2026-07-31T15:11:05Z" message: 'Created pod: success-200-isvc-06a6c-predictor-69d6f4dc94-zf296' metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:11:05Z" name: success-200-isvc-06a6c-predictor-69d6f4dc94.18c7687d75790e34 namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: c0286cb7-eb2a-497e-a14b-7d9c25313b00 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-06a6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: 765b6abe-9136-46a1-8c1d-2a69e155596d kind: Event lastTimestamp: "2026-07-31T15:11:05Z" message: Scaled up replica set success-200-isvc-06a6c-predictor-69d6f4dc94 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:11:05Z" name: success-200-isvc-06a6c-predictor.18c7687d72950952 namespace: kserve-ci-e2e-test resourceVersion: "21496" uid: c7843f5f-da5d-448d-b7fd-de7a31214a89 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:11:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-06a6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: 6fdf05f9-56d4-46f2-a99b-b6e617f2063e kind: Event lastTimestamp: "2026-07-31T15:11:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:11:50Z" name: success-200-isvc-06a6c-predictor.18c76880f14a9684 namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: 973d78a3-034b-4888-8c65-e3ed77332f90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:11:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-06a6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: 6fdf05f9-56d4-46f2-a99b-b6e617f2063e kind: Event lastTimestamp: "2026-07-31T15:11:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:11:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:11:50Z" name: success-200-isvc-06a6c-predictor.18c76880f1b92229 namespace: kserve-ci-e2e-test resourceVersion: "21977" uid: a80177d3-b08f-4ea7-b654-0523a3a982d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-06a6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21721" uid: 6fdf05f9-56d4-46f2-a99b-b6e617f2063e kind: Event lastTimestamp: "2026-07-31T15:12:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:20Z" name: success-200-isvc-06a6c-predictor.18c7688b6e950555 namespace: kserve-ci-e2e-test resourceVersion: "22453" uid: 4dfe325d-ff0b-409d-a1ab-0c83c541bacf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-06a6c-predictor namespace: kserve-ci-e2e-test resourceVersion: "21721" uid: 6fdf05f9-56d4-46f2-a99b-b6e617f2063e kind: Event lastTimestamp: "2026-07-31T15:12:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:20Z" name: success-200-isvc-06a6c-predictor.18c7688b6edd2df8 namespace: kserve-ci-e2e-test resourceVersion: "22454" uid: e251a1f6-6b73-4f40-a503-8601f68ca42c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-06a6c namespace: kserve-ci-e2e-test resourceVersion: "21471" uid: faa122a8-8739-4e80-a367-29c797191940 kind: Event lastTimestamp: "2026-07-31T15:11:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:11:52Z" name: success-200-isvc-06a6c.18c7687d72d0f2d7 namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: ab0ff3c5-aa93-4809-9b37-8fe623e7f7b0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-06a6c namespace: kserve-ci-e2e-test resourceVersion: "22012" uid: faa122a8-8739-4e80-a367-29c797191940 kind: Event lastTimestamp: "2026-07-31T15:11:52Z" message: InferenceService [success-200-isvc-06a6c] is Ready metadata: creationTimestamp: "2026-07-31T15:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:11:52Z" name: success-200-isvc-06a6c.18c768887ea72d86 namespace: kserve-ci-e2e-test resourceVersion: "22013" uid: 4107b015-9090-4d37-8a55-cc50eab1d795 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:09:54.986235Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-49957-predictor-c6d6b9746-75srq namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 7b69d458-b631-4a3c-a66f-f7f5b63d59b3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-49957-predictor-c6d6b9746-75srq to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:09:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:09:54Z" name: success-200-isvc-49957-predictor-c6d6b9746-75srq.18c7686d14d9f7f5 namespace: kserve-ci-e2e-test resourceVersion: "20624" uid: f9287911-5fd1-41ba-8759-9b2ebdd23a36 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-49957-predictor-c6d6b9746-75srq namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: 7b69d458-b631-4a3c-a66f-f7f5b63d59b3 kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:09:55Z" name: success-200-isvc-49957-predictor-c6d6b9746-75srq.18c7686d2f36d769 namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: 620222a2-2066-446a-854a-5371a465116e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49957-predictor-c6d6b9746-75srq namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 7b69d458-b631-4a3c-a66f-f7f5b63d59b3 kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: success-200-isvc-49957-predictor-c6d6b9746-75srq.18c7686d30a8c820 namespace: kserve-ci-e2e-test resourceVersion: "20681" uid: 21b9e08f-fdb7-4dd7-8c4f-6712eb6f4c3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49957-predictor-c6d6b9746-75srq namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 7b69d458-b631-4a3c-a66f-f7f5b63d59b3 kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: success-200-isvc-49957-predictor-c6d6b9746-75srq.18c7686d34303eea namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 5e69a00d-6359-4c10-ba8c-08ff25397ce4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49957-predictor-c6d6b9746-75srq namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 7b69d458-b631-4a3c-a66f-f7f5b63d59b3 kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: success-200-isvc-49957-predictor-c6d6b9746-75srq.18c7686d34bfca2d namespace: kserve-ci-e2e-test resourceVersion: "20687" uid: 5a0411fa-aa89-42bf-9b81-12a8f6004e31 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49957-predictor-c6d6b9746-75srq namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 7b69d458-b631-4a3c-a66f-f7f5b63d59b3 kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: success-200-isvc-49957-predictor-c6d6b9746-75srq.18c7686d34cd1d05 namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: b667ff1b-fca3-44c8-9435-c3a91eb216a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49957-predictor-c6d6b9746-75srq namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 7b69d458-b631-4a3c-a66f-f7f5b63d59b3 kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: success-200-isvc-49957-predictor-c6d6b9746-75srq.18c7686d37e2c156 namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: d18f6f8b-3b28-46c6-91d6-1c1c0c6f6269 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49957-predictor-c6d6b9746-75srq namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 7b69d458-b631-4a3c-a66f-f7f5b63d59b3 kind: Event lastTimestamp: "2026-07-31T15:09:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:09:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:55Z" name: success-200-isvc-49957-predictor-c6d6b9746-75srq.18c7686d3864aa31 namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: 1c849991-e90c-460f-aab5-d1d48310a8f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:09:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49957-predictor-c6d6b9746-75srq namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 7b69d458-b631-4a3c-a66f-f7f5b63d59b3 kind: Event lastTimestamp: "2026-07-31T15:10:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:09:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:32Z" name: success-200-isvc-49957-predictor-c6d6b9746-75srq.18c7686d648f96cd namespace: kserve-ci-e2e-test resourceVersion: "20996" uid: 0131638f-4e40-4e9c-ab1d-72e4ab7b5667 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49957-predictor-c6d6b9746-75srq namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 7b69d458-b631-4a3c-a66f-f7f5b63d59b3 kind: Event lastTimestamp: "2026-07-31T15:11:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:05Z" name: success-200-isvc-49957-predictor-c6d6b9746-75srq.18c7687d70ad67b4 namespace: kserve-ci-e2e-test resourceVersion: "21487" uid: 691b62e3-693f-4cc9-b6aa-1e17a64f3e04 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49957-predictor-c6d6b9746-75srq namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 7b69d458-b631-4a3c-a66f-f7f5b63d59b3 kind: Event lastTimestamp: "2026-07-31T15:11:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:05Z" name: success-200-isvc-49957-predictor-c6d6b9746-75srq.18c7687d70adf357 namespace: kserve-ci-e2e-test resourceVersion: "21490" uid: d390ec6e-efb3-4be5-aa38-b77677f68fb1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49957-predictor-c6d6b9746-75srq namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 7b69d458-b631-4a3c-a66f-f7f5b63d59b3 kind: Event lastTimestamp: "2026-07-31T15:11:07Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:11:07Z" name: success-200-isvc-49957-predictor-c6d6b9746-75srq.18c7687deccf45b1 namespace: kserve-ci-e2e-test resourceVersion: "21612" uid: d2282021-28eb-4ae7-9a3b-29c97f6db020 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-49957-predictor-c6d6b9746 namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 8f5a9fc6-e8af-46e6-b80b-6d0d25b116ad kind: Event lastTimestamp: "2026-07-31T15:09:54Z" message: 'Created pod: success-200-isvc-49957-predictor-c6d6b9746-75srq' metadata: creationTimestamp: "2026-07-31T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:09:54Z" name: success-200-isvc-49957-predictor-c6d6b9746.18c7686d146ab935 namespace: kserve-ci-e2e-test resourceVersion: "20619" uid: 172bceb4-4eca-447c-99d4-1ddd253ab69e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-49957-predictor namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 2de5d4f8-58e0-4421-8681-4744a0adba33 kind: Event lastTimestamp: "2026-07-31T15:09:54Z" message: Scaled up replica set success-200-isvc-49957-predictor-c6d6b9746 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:09:54Z" name: success-200-isvc-49957-predictor.18c7686d10753638 namespace: kserve-ci-e2e-test resourceVersion: "20602" uid: 2fd5a34d-d8d1-48d4-a69f-923e52758248 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:10:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-49957-predictor namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 9907edb4-e481-44f2-a8e6-2c4ab74b06b6 kind: Event lastTimestamp: "2026-07-31T15:10:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:54Z" name: success-200-isvc-49957-predictor.18c768708ef5bf95 namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 07336a3f-439b-4622-b783-0f48c17257d0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:10:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-49957-predictor namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 9907edb4-e481-44f2-a8e6-2c4ab74b06b6 kind: Event lastTimestamp: "2026-07-31T15:10:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:10:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:54Z" name: success-200-isvc-49957-predictor.18c768708f67923c namespace: kserve-ci-e2e-test resourceVersion: "21327" uid: 0a4cc671-6791-487a-bc52-5c62394cdd8b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-49957-predictor namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: 9907edb4-e481-44f2-a8e6-2c4ab74b06b6 kind: Event lastTimestamp: "2026-07-31T15:10:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:24Z" name: success-200-isvc-49957-predictor.18c768740e1a6e02 namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: dd141f65-5f06-47de-b046-471f25e4e45a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-49957-predictor namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: 9907edb4-e481-44f2-a8e6-2c4ab74b06b6 kind: Event lastTimestamp: "2026-07-31T15:10:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:10:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:24Z" name: success-200-isvc-49957-predictor.18c768740e8a9926 namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: 617eb181-f2e4-4527-a1fe-9c4be2e17081 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:09:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-49957 namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: a61b6f7a-cf32-4389-ae33-fda630a803a0 kind: Event lastTimestamp: "2026-07-31T15:10:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:10:42Z" name: success-200-isvc-49957.18c7686d1091dbcd namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: b000c538-4e03-4c01-802e-87ae07b38925 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-49957 namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: a61b6f7a-cf32-4389-ae33-fda630a803a0 kind: Event lastTimestamp: "2026-07-31T15:09:54Z" message: 'Failed to update status for InferenceService "success-200-isvc-49957": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-49957": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:09:54Z" name: success-200-isvc-49957.18c7686d1168b474 namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 8ce3deae-8e11-4896-9d25-1579bac6f591 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-49957 namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: a61b6f7a-cf32-4389-ae33-fda630a803a0 kind: Event lastTimestamp: "2026-07-31T15:09:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-49957": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:09:54Z" name: success-200-isvc-49957.18c7686d116917a2 namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: 6c619a83-5e51-45e7-9266-f963e976cfa7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-49957 namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: a61b6f7a-cf32-4389-ae33-fda630a803a0 kind: Event lastTimestamp: "2026-07-31T15:10:42Z" message: InferenceService [success-200-isvc-49957] is Ready metadata: creationTimestamp: "2026-07-31T15:10:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:10:42Z" name: success-200-isvc-49957.18c768782036c1ce namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 4eb50efa-c12c-4a77-9521-912e0f4136bf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:21:30.340344Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg namespace: kserve-ci-e2e-test resourceVersion: "26915" uid: 3460efc7-de17-47da-89e9-87bcba812bbe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7902a-predictor-8dd58f79b-bjhqg to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg.18c7690efb2fab35 namespace: kserve-ci-e2e-test resourceVersion: "26923" uid: e4b0b760-1240-4101-94f3-7231498bf07c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg namespace: kserve-ci-e2e-test resourceVersion: "26934" uid: 3460efc7-de17-47da-89e9-87bcba812bbe kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg.18c7690f1522d836 namespace: kserve-ci-e2e-test resourceVersion: "26995" uid: 78e8c4b1-00a5-4d8a-b142-a3ee4c2392aa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 3460efc7-de17-47da-89e9-87bcba812bbe kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg.18c7690f16e2f956 namespace: kserve-ci-e2e-test resourceVersion: "27003" uid: 7ac5595b-479b-4c93-a5ea-b7ef75be1999 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 3460efc7-de17-47da-89e9-87bcba812bbe kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg.18c7690f1a892b05 namespace: kserve-ci-e2e-test resourceVersion: "27016" uid: 5d5830ac-9c33-45c5-9ee0-1400ba849877 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 3460efc7-de17-47da-89e9-87bcba812bbe kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg.18c7690f1b393933 namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: 67b5193b-b404-4fdd-b00d-e3202de2df6f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 3460efc7-de17-47da-89e9-87bcba812bbe kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg.18c7690f1b45ab27 namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: 3e604704-8ee3-4ef6-93c2-0b2b1bea5461 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 3460efc7-de17-47da-89e9-87bcba812bbe kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg.18c7690f1f588f1e namespace: kserve-ci-e2e-test resourceVersion: "27041" uid: e8c45da7-fc71-43e4-b300-1485f559f0dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 3460efc7-de17-47da-89e9-87bcba812bbe kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg.18c7690f1ff1c3f5 namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: ee4ec8b6-bc2c-4187-9594-5782cdc19814 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 3460efc7-de17-47da-89e9-87bcba812bbe kind: Event lastTimestamp: "2026-07-31T15:22:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:22:07Z" name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg.18c7690f217906ad namespace: kserve-ci-e2e-test resourceVersion: "27455" uid: fc088d2c-2db2-4b24-99e1-75e62963e4e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 3460efc7-de17-47da-89e9-87bcba812bbe kind: Event lastTimestamp: "2026-07-31T15:38:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:38:49Z" name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg.18c76a00ff8a813c namespace: kserve-ci-e2e-test resourceVersion: "34548" uid: 17b79add-63a3-499f-a145-319b51a040ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 3460efc7-de17-47da-89e9-87bcba812bbe kind: Event lastTimestamp: "2026-07-31T15:38:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:38:49Z" name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg.18c76a00ff8afdaa namespace: kserve-ci-e2e-test resourceVersion: "34549" uid: ee385064-e23d-4a1c-b1ae-fedef0bfbbdc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:38:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 3460efc7-de17-47da-89e9-87bcba812bbe kind: Event lastTimestamp: "2026-07-31T15:38:52Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:38:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:38:52Z" name: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg.18c76a01838e1a36 namespace: kserve-ci-e2e-test resourceVersion: "34614" uid: 0c380edc-3e9e-4668-ac7c-01fc864cf4db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7902a-predictor-8dd58f79b namespace: kserve-ci-e2e-test resourceVersion: "26890" uid: 9df0377c-55a5-4b2e-98ce-1589a349f045 kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: 'Created pod: success-200-isvc-7902a-predictor-8dd58f79b-bjhqg' metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7902a-predictor-8dd58f79b.18c7690efaaebc62 namespace: kserve-ci-e2e-test resourceVersion: "26919" uid: 46b0a064-8ce8-4fc4-a28d-952c62c83e93 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7902a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: 464b4f98-a6ab-4857-a059-214b32e279b8 kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Scaled up replica set success-200-isvc-7902a-predictor-8dd58f79b from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7902a-predictor.18c7690ef731821d namespace: kserve-ci-e2e-test resourceVersion: "26898" uid: e7f01d8a-e905-46a5-a4d9-4d2e2cfd0f6b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:21:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7902a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26889" uid: bdccc8ce-9637-44b0-8b21-1a21e8821c5e kind: Event lastTimestamp: "2026-07-31T15:22:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:21:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:00Z" name: success-200-isvc-7902a-predictor.18c76912764053f6 namespace: kserve-ci-e2e-test resourceVersion: "27395" uid: 75ddee14-e5c1-4fc7-aba0-f3dadac001a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:21:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7902a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26889" uid: bdccc8ce-9637-44b0-8b21-1a21e8821c5e kind: Event lastTimestamp: "2026-07-31T15:22:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:21:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:00Z" name: success-200-isvc-7902a-predictor.18c7691276c664f7 namespace: kserve-ci-e2e-test resourceVersion: "27397" uid: d9a5c0b9-2173-469d-9dbe-a9b93121f747 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:22:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7902a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27265" uid: bdccc8ce-9637-44b0-8b21-1a21e8821c5e kind: Event lastTimestamp: "2026-07-31T15:22:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:45Z" name: success-200-isvc-7902a-predictor.18c7691974f07df3 namespace: kserve-ci-e2e-test resourceVersion: "27857" uid: e1445b84-2e0f-493a-86fd-a30b8d176f4c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:22:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7902a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27265" uid: bdccc8ce-9637-44b0-8b21-1a21e8821c5e kind: Event lastTimestamp: "2026-07-31T15:22:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:45Z" name: success-200-isvc-7902a-predictor.18c769197564f9d5 namespace: kserve-ci-e2e-test resourceVersion: "27858" uid: 58caf4a5-3eca-4cfb-9d2b-8c0c38bddb50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7902a namespace: kserve-ci-e2e-test resourceVersion: "26868" uid: 5f8d1ecf-3547-4bb4-9ff1-c48e6440b4ed kind: Event lastTimestamp: "2026-07-31T15:22:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:22:17Z" name: success-200-isvc-7902a.18c7690ef7720005 namespace: kserve-ci-e2e-test resourceVersion: "27550" uid: 0dccf14c-e1e2-4c06-833d-8f51ffff006b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7902a namespace: kserve-ci-e2e-test resourceVersion: "26868" uid: 5f8d1ecf-3547-4bb4-9ff1-c48e6440b4ed kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: 'Failed to update status for InferenceService "success-200-isvc-7902a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7902a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7902a.18c7690ef8753a70 namespace: kserve-ci-e2e-test resourceVersion: "26906" uid: f8c3202c-932c-45d0-b774-409b4440417b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7902a namespace: kserve-ci-e2e-test resourceVersion: "26868" uid: 5f8d1ecf-3547-4bb4-9ff1-c48e6440b4ed kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7902a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7902a.18c7690ef8758572 namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: 268291cb-8f72-452f-8745-95ac6dd8bad9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7902a namespace: kserve-ci-e2e-test resourceVersion: "27548" uid: 5f8d1ecf-3547-4bb4-9ff1-c48e6440b4ed kind: Event lastTimestamp: "2026-07-31T15:22:17Z" message: InferenceService [success-200-isvc-7902a] is Ready metadata: creationTimestamp: "2026-07-31T15:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:22:17Z" name: success-200-isvc-7902a.18c76919e1851f28 namespace: kserve-ci-e2e-test resourceVersion: "27549" uid: 2b6ba8de-40c7-4ae5-87d0-65b8da4a7ea8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:00:40.210591Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2 namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: b9022088-6ae0-450a-b0e6-abafe9638ffb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7b0b5-predictor-df99ffc46-svnw2 to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2.18c767ebe9a56adc namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 250b0aed-f153-4d0a-b1a1-ae770e2631b1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2 namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: b9022088-6ae0-450a-b0e6-abafe9638ffb kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2.18c767ec039bc477 namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: a313f24f-3b1d-415a-924c-269405d0af8d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: b9022088-6ae0-450a-b0e6-abafe9638ffb kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2.18c767ec05114987 namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: 0bfde8fd-9c34-4faf-906f-cf24824727df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: b9022088-6ae0-450a-b0e6-abafe9638ffb kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2.18c767ec090d7c6a namespace: kserve-ci-e2e-test resourceVersion: "16292" uid: 40e22032-9029-4572-be3e-8d62b0201551 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: b9022088-6ae0-450a-b0e6-abafe9638ffb kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2.18c767ec098dbd51 namespace: kserve-ci-e2e-test resourceVersion: "16294" uid: d9bfb989-8725-441b-a1d3-c6af22e2630f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: b9022088-6ae0-450a-b0e6-abafe9638ffb kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2.18c767ec09981acc namespace: kserve-ci-e2e-test resourceVersion: "16296" uid: 85bad0de-fa75-4a8c-8aca-8eac0fad7714 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: b9022088-6ae0-450a-b0e6-abafe9638ffb kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2.18c767ec0d70290b namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 3b186655-94df-489b-bb28-3fbd30a3ca9b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: b9022088-6ae0-450a-b0e6-abafe9638ffb kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2.18c767ec0de99fee namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: c0e2b0c3-9d9e-4d18-a127-0b264a804b91 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T15:00:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: b9022088-6ae0-450a-b0e6-abafe9638ffb kind: Event lastTimestamp: "2026-07-31T15:09:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:58Z" name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2.18c767ec6386c89e namespace: kserve-ci-e2e-test resourceVersion: "20730" uid: 9fc9cbf9-3fe8-4e3c-8bbe-fd7968efc492 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: b9022088-6ae0-450a-b0e6-abafe9638ffb kind: Event lastTimestamp: "2026-07-31T15:09:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:54Z" name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2.18c7686d126e7d66 namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: 94f0e249-6663-4cf5-a6d1-773b269489ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: b9022088-6ae0-450a-b0e6-abafe9638ffb kind: Event lastTimestamp: "2026-07-31T15:09:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:54Z" name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2.18c7686d12700bb7 namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: 0118e724-b90c-4618-9fbb-164609d08e7e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: b9022088-6ae0-450a-b0e6-abafe9638ffb kind: Event lastTimestamp: "2026-07-31T15:09:58Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:09:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:58Z" name: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2.18c7686dd80f68ec namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: f7f5b1e5-d16a-4c67-9c47-64d3a91c876d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7b0b5-predictor-df99ffc46 namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: a3f2ebf8-0577-4376-b320-f3069b594e85 kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: 'Created pod: success-200-isvc-7b0b5-predictor-df99ffc46-svnw2' metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-7b0b5-predictor-df99ffc46.18c767ebe8fa7a47 namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: 5eafce45-b13e-4736-b8e7-de7dcd66325c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7b0b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: ab52aa69-ccb0-45f7-858e-5d0b82e7368a kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Scaled up replica set success-200-isvc-7b0b5-predictor-df99ffc46 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-7b0b5-predictor.18c767ebe45391fb namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: 8d14eedc-961b-4984-a0a9-df3d1ce4b5fd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T15:00:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b0b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: a80cb56d-8c2f-43cd-b584-9a430e6ad3ae kind: Event lastTimestamp: "2026-07-31T15:01:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:00:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:55Z" name: success-200-isvc-7b0b5-predictor.18c767ef62d0d9bf namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: 1d207ee3-4c16-4055-82b4-67cceb54691b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T15:00:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b0b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: a80cb56d-8c2f-43cd-b584-9a430e6ad3ae kind: Event lastTimestamp: "2026-07-31T15:01:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:00:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:55Z" name: success-200-isvc-7b0b5-predictor.18c767ef633ee2b8 namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: 50594c5b-9805-452d-b10e-e67a8c37c3a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b0b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: a80cb56d-8c2f-43cd-b584-9a430e6ad3ae kind: Event lastTimestamp: "2026-07-31T15:01:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:10Z" name: success-200-isvc-7b0b5-predictor.18c767f2e207d216 namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: 18cdf4d2-7ebf-4398-8c7d-4c1904e799b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b0b5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: a80cb56d-8c2f-43cd-b584-9a430e6ad3ae kind: Event lastTimestamp: "2026-07-31T15:01:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:10Z" name: success-200-isvc-7b0b5-predictor.18c767f2e27d15a8 namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: c66895ec-53c7-4625-87de-6f7135f4179a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7b0b5 namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: 80ffdc60-4434-46cb-b40e-f056197c0401 kind: Event lastTimestamp: "2026-07-31T15:01:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:01:28Z" name: success-200-isvc-7b0b5.18c767ebe45888fc namespace: kserve-ci-e2e-test resourceVersion: "16788" uid: 60f72705-f78d-4189-923c-c05a28e929b1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7b0b5 namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: 80ffdc60-4434-46cb-b40e-f056197c0401 kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: 'Failed to update status for InferenceService "success-200-isvc-7b0b5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7b0b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-7b0b5.18c767ebe502ca49 namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: 9aab1a1c-b940-498e-9dbd-130411262b6f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7b0b5 namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: 80ffdc60-4434-46cb-b40e-f056197c0401 kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7b0b5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-7b0b5.18c767ebe5030581 namespace: kserve-ci-e2e-test resourceVersion: "16205" uid: bac3f62f-d84d-40c1-a593-c93b173719c4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7b0b5 namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: 80ffdc60-4434-46cb-b40e-f056197c0401 kind: Event lastTimestamp: "2026-07-31T15:01:28Z" message: InferenceService [success-200-isvc-7b0b5] is Ready metadata: creationTimestamp: "2026-07-31T15:01:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:01:28Z" name: success-200-isvc-7b0b5.18c767f71ec3b4d7 namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: b22ff5d2-93ac-46a3-9611-37f05943a890 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:12:15.312731Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7b982-predictor-5857f8478-6d7w5 namespace: kserve-ci-e2e-test resourceVersion: "22323" uid: 5afd39db-ff97-48a5-85d6-9b9266fa58b2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7b982-predictor-5857f8478-6d7w5 to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:12:15Z" name: success-200-isvc-7b982-predictor-5857f8478-6d7w5.18c7688dc0f662ea namespace: kserve-ci-e2e-test resourceVersion: "22333" uid: e80638c4-c9aa-49ed-b37c-aa0b4acb3bd2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7b982-predictor-5857f8478-6d7w5 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: 5afd39db-ff97-48a5-85d6-9b9266fa58b2 kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7b982-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:15Z" name: success-200-isvc-7b982-predictor-5857f8478-6d7w5.18c7688dce00e9a0 namespace: kserve-ci-e2e-test resourceVersion: "22376" uid: 62998dcc-c024-4d23-b6af-6bcb67040cb8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7b982-predictor-5857f8478-6d7w5 namespace: kserve-ci-e2e-test resourceVersion: "22345" uid: 5afd39db-ff97-48a5-85d6-9b9266fa58b2 kind: Event lastTimestamp: "2026-07-31T15:12:16Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:12:16Z" name: success-200-isvc-7b982-predictor-5857f8478-6d7w5.18c7688dff59932d namespace: kserve-ci-e2e-test resourceVersion: "22394" uid: 879ec8f7-5b7f-4af2-8a86-04c35cc514d3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b982-predictor-5857f8478-6d7w5 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: 5afd39db-ff97-48a5-85d6-9b9266fa58b2 kind: Event lastTimestamp: "2026-07-31T15:12:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:16Z" name: success-200-isvc-7b982-predictor-5857f8478-6d7w5.18c7688e00b73f18 namespace: kserve-ci-e2e-test resourceVersion: "22396" uid: 2ca84cb7-dac8-48a7-8c8d-ca3546105773 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b982-predictor-5857f8478-6d7w5 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: 5afd39db-ff97-48a5-85d6-9b9266fa58b2 kind: Event lastTimestamp: "2026-07-31T15:12:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:16Z" name: success-200-isvc-7b982-predictor-5857f8478-6d7w5.18c7688e03b25e13 namespace: kserve-ci-e2e-test resourceVersion: "22397" uid: 70adb7e3-5a80-46b6-8ee7-c67d5b398664 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b982-predictor-5857f8478-6d7w5 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: 5afd39db-ff97-48a5-85d6-9b9266fa58b2 kind: Event lastTimestamp: "2026-07-31T15:12:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:16Z" name: success-200-isvc-7b982-predictor-5857f8478-6d7w5.18c7688e042715a2 namespace: kserve-ci-e2e-test resourceVersion: "22398" uid: 6910da3c-9874-4677-bbb7-92f26c701220 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b982-predictor-5857f8478-6d7w5 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: 5afd39db-ff97-48a5-85d6-9b9266fa58b2 kind: Event lastTimestamp: "2026-07-31T15:12:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:16Z" name: success-200-isvc-7b982-predictor-5857f8478-6d7w5.18c7688e04324d0f namespace: kserve-ci-e2e-test resourceVersion: "22399" uid: 6365348f-97d5-427c-b57b-2c6c55593a62 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b982-predictor-5857f8478-6d7w5 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: 5afd39db-ff97-48a5-85d6-9b9266fa58b2 kind: Event lastTimestamp: "2026-07-31T15:12:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:16Z" name: success-200-isvc-7b982-predictor-5857f8478-6d7w5.18c7688e074e95bd namespace: kserve-ci-e2e-test resourceVersion: "22400" uid: 118f2ebf-b228-4724-bb26-82b87483952f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b982-predictor-5857f8478-6d7w5 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: 5afd39db-ff97-48a5-85d6-9b9266fa58b2 kind: Event lastTimestamp: "2026-07-31T15:12:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:16Z" name: success-200-isvc-7b982-predictor-5857f8478-6d7w5.18c7688e07ffa735 namespace: kserve-ci-e2e-test resourceVersion: "22401" uid: ad49b6b9-8971-4677-b3fc-a831b0f54f7c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T15:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b982-predictor-5857f8478-6d7w5 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: 5afd39db-ff97-48a5-85d6-9b9266fa58b2 kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: success-200-isvc-7b982-predictor-5857f8478-6d7w5.18c7688e20e62a1e namespace: kserve-ci-e2e-test resourceVersion: "27141" uid: ef83584b-cbce-40c3-9731-025313793b94 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b982-predictor-5857f8478-6d7w5 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: 5afd39db-ff97-48a5-85d6-9b9266fa58b2 kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7b982-predictor-5857f8478-6d7w5.18c7690ef6dced92 namespace: kserve-ci-e2e-test resourceVersion: "26896" uid: d1414cd9-c842-41e2-8b0b-5e9179fa8ffc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b982-predictor-5857f8478-6d7w5 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: 5afd39db-ff97-48a5-85d6-9b9266fa58b2 kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-7b982-predictor-5857f8478-6d7w5.18c7690ef6dd906c namespace: kserve-ci-e2e-test resourceVersion: "26900" uid: 2c9504a2-52ba-457d-ace3-0806ed3a361b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b982-predictor-5857f8478-6d7w5 namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: 5afd39db-ff97-48a5-85d6-9b9266fa58b2 kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: success-200-isvc-7b982-predictor-5857f8478-6d7w5.18c7690f95502f75 namespace: kserve-ci-e2e-test resourceVersion: "27139" uid: 3ae58ef1-172c-4e33-a2f4-7d51fb91817e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7b982-predictor-5857f8478 namespace: kserve-ci-e2e-test resourceVersion: "22307" uid: 7f8c9e4e-13d0-4be1-babb-956de1792624 kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: 'Created pod: success-200-isvc-7b982-predictor-5857f8478-6d7w5' metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:15Z" name: success-200-isvc-7b982-predictor-5857f8478.18c7688dc06d7749 namespace: kserve-ci-e2e-test resourceVersion: "22328" uid: ed64e73b-f1c1-4e36-9c74-bc6159f36eac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7b982-predictor namespace: kserve-ci-e2e-test resourceVersion: "22301" uid: 7df13273-0e61-4960-aeb5-a58d46420937 kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: Scaled up replica set success-200-isvc-7b982-predictor-5857f8478 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:15Z" name: success-200-isvc-7b982-predictor.18c7688dbe0531d4 namespace: kserve-ci-e2e-test resourceVersion: "22309" uid: fe723e91-8277-4bb9-ac1a-2df246697cfd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:12:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b982-predictor namespace: kserve-ci-e2e-test resourceVersion: "22305" uid: 3480c73d-36fc-4efc-8d6e-1dcf1933eb83 kind: Event lastTimestamp: "2026-07-31T15:12:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:45Z" name: success-200-isvc-7b982-predictor.18c768913cf36b58 namespace: kserve-ci-e2e-test resourceVersion: "22652" uid: 8762a28c-8eb9-4360-b0b6-0d58152ba0ce reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:12:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b982-predictor namespace: kserve-ci-e2e-test resourceVersion: "22305" uid: 3480c73d-36fc-4efc-8d6e-1dcf1933eb83 kind: Event lastTimestamp: "2026-07-31T15:12:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:12:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:45Z" name: success-200-isvc-7b982-predictor.18c768913d48f55e namespace: kserve-ci-e2e-test resourceVersion: "22653" uid: 4a29f1d2-48ba-41e6-a8de-6554cc3b4562 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:13:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b982-predictor namespace: kserve-ci-e2e-test resourceVersion: "22533" uid: 3480c73d-36fc-4efc-8d6e-1dcf1933eb83 kind: Event lastTimestamp: "2026-07-31T15:13:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:13:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:13:30Z" name: success-200-isvc-7b982-predictor.18c768983ab572ce namespace: kserve-ci-e2e-test resourceVersion: "23038" uid: b7dd26aa-a4bb-4190-94d2-65565704d8db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:13:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b982-predictor namespace: kserve-ci-e2e-test resourceVersion: "22533" uid: 3480c73d-36fc-4efc-8d6e-1dcf1933eb83 kind: Event lastTimestamp: "2026-07-31T15:13:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:13:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:13:30Z" name: success-200-isvc-7b982-predictor.18c768983b178f9c namespace: kserve-ci-e2e-test resourceVersion: "23039" uid: f717b691-f3b0-414b-ac6d-f14087fa3280 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7b982 namespace: kserve-ci-e2e-test resourceVersion: "22286" uid: 665f54b4-4bcf-4b4c-b0d6-fa07df270362 kind: Event lastTimestamp: "2026-07-31T15:13:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:13:03Z" name: success-200-isvc-7b982.18c7688dbe0901cf namespace: kserve-ci-e2e-test resourceVersion: "22807" uid: 642a4e81-e69e-43ad-b3bc-8cf06c2a9718 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:13:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7b982 namespace: kserve-ci-e2e-test resourceVersion: "22801" uid: 665f54b4-4bcf-4b4c-b0d6-fa07df270362 kind: Event lastTimestamp: "2026-07-31T15:13:03Z" message: InferenceService [success-200-isvc-7b982] is Ready metadata: creationTimestamp: "2026-07-31T15:13:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:13:03Z" name: success-200-isvc-7b982.18c76898dfb4e62d namespace: kserve-ci-e2e-test resourceVersion: "22803" uid: 2ae1ab12-3670-4b9e-8aec-a9f24fd334f3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:10:50.931653Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a1442-predictor-5759c7c998-p46m5 namespace: kserve-ci-e2e-test resourceVersion: "21213" uid: 69745f64-c3d6-4e6f-9780-54ca490f1ee2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a1442-predictor-5759c7c998-p46m5 to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:10:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:10:50Z" name: success-200-isvc-a1442-predictor-5759c7c998-p46m5.18c7687a1b755415 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: e7017659-f063-49dd-ac83-2f20752ae0b0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a1442-predictor-5759c7c998-p46m5 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 69745f64-c3d6-4e6f-9780-54ca490f1ee2 kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a1442-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:51Z" name: success-200-isvc-a1442-predictor-5759c7c998-p46m5.18c7687a2308de5d namespace: kserve-ci-e2e-test resourceVersion: "21249" uid: 87493595-6d1b-4a97-a1bf-06df7bab3ff4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a1442-predictor-5759c7c998-p46m5 namespace: kserve-ci-e2e-test resourceVersion: "21224" uid: 69745f64-c3d6-4e6f-9780-54ca490f1ee2 kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:10:51Z" name: success-200-isvc-a1442-predictor-5759c7c998-p46m5.18c7687a596dafd6 namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 20066f2e-7f22-40c1-92e5-b81be2ef5f89 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a1442-predictor-5759c7c998-p46m5 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 69745f64-c3d6-4e6f-9780-54ca490f1ee2 kind: Event lastTimestamp: "2026-07-31T15:10:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:51Z" name: success-200-isvc-a1442-predictor-5759c7c998-p46m5.18c7687a5acde48c namespace: kserve-ci-e2e-test resourceVersion: "21281" uid: 7ed16e09-3d92-454b-a2bf-5713ab46e9a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a1442-predictor-5759c7c998-p46m5 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 69745f64-c3d6-4e6f-9780-54ca490f1ee2 kind: Event lastTimestamp: "2026-07-31T15:10:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:52Z" name: success-200-isvc-a1442-predictor-5759c7c998-p46m5.18c7687a5dc5c3ae namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: f4d6495b-6776-4828-945d-714d994fb9c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a1442-predictor-5759c7c998-p46m5 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 69745f64-c3d6-4e6f-9780-54ca490f1ee2 kind: Event lastTimestamp: "2026-07-31T15:10:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:52Z" name: success-200-isvc-a1442-predictor-5759c7c998-p46m5.18c7687a5e3c6155 namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: 10c8703c-875b-426b-be19-695785d491d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a1442-predictor-5759c7c998-p46m5 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 69745f64-c3d6-4e6f-9780-54ca490f1ee2 kind: Event lastTimestamp: "2026-07-31T15:10:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:52Z" name: success-200-isvc-a1442-predictor-5759c7c998-p46m5.18c7687a5e46cc21 namespace: kserve-ci-e2e-test resourceVersion: "21284" uid: c023b888-7f78-4fa8-b9fb-62154a6f0082 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a1442-predictor-5759c7c998-p46m5 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 69745f64-c3d6-4e6f-9780-54ca490f1ee2 kind: Event lastTimestamp: "2026-07-31T15:10:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:52Z" name: success-200-isvc-a1442-predictor-5759c7c998-p46m5.18c7687a61abfd4e namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: c935dfec-a4a2-4ac1-8bb9-645a378d7386 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a1442-predictor-5759c7c998-p46m5 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 69745f64-c3d6-4e6f-9780-54ca490f1ee2 kind: Event lastTimestamp: "2026-07-31T15:10:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:52Z" name: success-200-isvc-a1442-predictor-5759c7c998-p46m5.18c7687a62292158 namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: 08e7a4dc-1df1-4db4-891f-bab8eed34c25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-31T15:10:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a1442-predictor-5759c7c998-p46m5 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 69745f64-c3d6-4e6f-9780-54ca490f1ee2 kind: Event lastTimestamp: "2026-07-31T15:12:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:10:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:18Z" name: success-200-isvc-a1442-predictor-5759c7c998-p46m5.18c7687a7c85b4e4 namespace: kserve-ci-e2e-test resourceVersion: "22431" uid: 7589e31a-4e75-4902-8146-06b4b18af2e3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a1442-predictor-5759c7c998-p46m5 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 69745f64-c3d6-4e6f-9780-54ca490f1ee2 kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:15Z" name: success-200-isvc-a1442-predictor-5759c7c998-p46m5.18c7688dbe310c9e namespace: kserve-ci-e2e-test resourceVersion: "22313" uid: b3361daf-87ac-4113-b5dc-9bdf867c6ebf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a1442-predictor-5759c7c998-p46m5 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 69745f64-c3d6-4e6f-9780-54ca490f1ee2 kind: Event lastTimestamp: "2026-07-31T15:12:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:12:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:15Z" name: success-200-isvc-a1442-predictor-5759c7c998-p46m5.18c7688dbe31a6c2 namespace: kserve-ci-e2e-test resourceVersion: "22318" uid: 79f200d1-4680-4688-9ef4-ae1b1ff285ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:12:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a1442-predictor-5759c7c998-p46m5 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 69745f64-c3d6-4e6f-9780-54ca490f1ee2 kind: Event lastTimestamp: "2026-07-31T15:12:18Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:12:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:12:18Z" name: success-200-isvc-a1442-predictor-5759c7c998-p46m5.18c7688e82e97d71 namespace: kserve-ci-e2e-test resourceVersion: "22429" uid: 4bf3930d-4a06-4627-9577-422a1f66d2db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a1442-predictor-5759c7c998 namespace: kserve-ci-e2e-test resourceVersion: "21197" uid: 79e6b93d-68e7-4828-bd8f-52d714a5928f kind: Event lastTimestamp: "2026-07-31T15:10:50Z" message: 'Created pod: success-200-isvc-a1442-predictor-5759c7c998-p46m5' metadata: creationTimestamp: "2026-07-31T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:50Z" name: success-200-isvc-a1442-predictor-5759c7c998.18c7687a1b06984e namespace: kserve-ci-e2e-test resourceVersion: "21215" uid: 211badbb-0d0f-4557-a55e-8732836e65b9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a1442-predictor namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: 632f993a-c97e-4ebc-9ddb-aa09f1fc7453 kind: Event lastTimestamp: "2026-07-31T15:10:50Z" message: Scaled up replica set success-200-isvc-a1442-predictor-5759c7c998 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:10:50Z" name: success-200-isvc-a1442-predictor.18c7687a18e908d8 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 6e6af5be-f543-4ea9-9463-1970a6467a33 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a1442-predictor namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: e58c1d89-b4a7-466f-aba5-c379e35afddb kind: Event lastTimestamp: "2026-07-31T15:11:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:11:35Z" name: success-200-isvc-a1442-predictor.18c7687d978e61bd namespace: kserve-ci-e2e-test resourceVersion: "21857" uid: 10687ab0-1286-46cb-bae6-ec9e0edc5a94 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:11:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a1442-predictor namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: e58c1d89-b4a7-466f-aba5-c379e35afddb kind: Event lastTimestamp: "2026-07-31T15:11:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:11:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:11:35Z" name: success-200-isvc-a1442-predictor.18c7687d981855de namespace: kserve-ci-e2e-test resourceVersion: "21858" uid: ddee896e-a59a-4f7e-9386-ac0601e1999f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:11:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a1442-predictor namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: e58c1d89-b4a7-466f-aba5-c379e35afddb kind: Event lastTimestamp: "2026-07-31T15:12:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:05Z" name: success-200-isvc-a1442-predictor.18c76888148fbc9f namespace: kserve-ci-e2e-test resourceVersion: "22188" uid: c89ca242-9663-46a7-bf65-406953ca432b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:11:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a1442-predictor namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: e58c1d89-b4a7-466f-aba5-c379e35afddb kind: Event lastTimestamp: "2026-07-31T15:12:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:12:05Z" name: success-200-isvc-a1442-predictor.18c7688814f901da namespace: kserve-ci-e2e-test resourceVersion: "22189" uid: d6bcb7e7-0c37-48b1-adc0-c74c56fd5781 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:10:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a1442 namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: 7c6d687b-7feb-4fc9-b2ea-8a8653eb27fd kind: Event lastTimestamp: "2026-07-31T15:11:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:11:48Z" name: success-200-isvc-a1442.18c7687a19200f99 namespace: kserve-ci-e2e-test resourceVersion: "21964" uid: 26bdcb76-4e98-400b-9f2a-702c63913efd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a1442 namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: 7c6d687b-7feb-4fc9-b2ea-8a8653eb27fd kind: Event lastTimestamp: "2026-07-31T15:10:50Z" message: 'Failed to update status for InferenceService "success-200-isvc-a1442": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a1442": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:10:50Z" name: success-200-isvc-a1442.18c7687a19f63fc1 namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: fdd7da00-2239-4597-9074-973cde4a2bbb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a1442 namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: 7c6d687b-7feb-4fc9-b2ea-8a8653eb27fd kind: Event lastTimestamp: "2026-07-31T15:10:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a1442": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:10:50Z" name: success-200-isvc-a1442.18c7687a19f68fad namespace: kserve-ci-e2e-test resourceVersion: "21212" uid: ba508698-45b9-4a7c-95fd-9b0b98856e11 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:11:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a1442 namespace: kserve-ci-e2e-test resourceVersion: "21958" uid: 7c6d687b-7feb-4fc9-b2ea-8a8653eb27fd kind: Event lastTimestamp: "2026-07-31T15:11:48Z" message: InferenceService [success-200-isvc-a1442] is Ready metadata: creationTimestamp: "2026-07-31T15:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:11:48Z" name: success-200-isvc-a1442.18c768878ccb2872 namespace: kserve-ci-e2e-test resourceVersion: "21959" uid: 58b623b0-ce65-4e14-bc38-b16ec5c59a70 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:01:35.985581Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6 namespace: kserve-ci-e2e-test resourceVersion: "16913" uid: 0512425d-0b38-4e37-9c0c-0fe6fbf49e17 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6 to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:01:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:01:35Z" name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6.18c767f8e618421b namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: d052e87c-5834-4c91-abbe-3dc7f5bbfb47 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6 namespace: kserve-ci-e2e-test resourceVersion: "16935" uid: 0512425d-0b38-4e37-9c0c-0fe6fbf49e17 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:01:36Z" name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6.18c767f8ffb0900a namespace: kserve-ci-e2e-test resourceVersion: "16984" uid: 032f38b9-6079-4c3a-88dc-516a5710798f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 0512425d-0b38-4e37-9c0c-0fe6fbf49e17 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6.18c767f90130cc0b namespace: kserve-ci-e2e-test resourceVersion: "16986" uid: faf61d2e-e714-4ee9-a26d-9e4f98e190a3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 0512425d-0b38-4e37-9c0c-0fe6fbf49e17 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6.18c767f904bb6324 namespace: kserve-ci-e2e-test resourceVersion: "16987" uid: a7392fc6-adaa-4723-b8b2-3eab6a270323 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 0512425d-0b38-4e37-9c0c-0fe6fbf49e17 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6.18c767f905424245 namespace: kserve-ci-e2e-test resourceVersion: "16989" uid: 994a9e4e-75b5-408b-b3b8-f106585ac31f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 0512425d-0b38-4e37-9c0c-0fe6fbf49e17 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6.18c767f9054d6b44 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: 47750abb-d423-4761-83e1-fff9df9f524c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 0512425d-0b38-4e37-9c0c-0fe6fbf49e17 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6.18c767f907e7d0c9 namespace: kserve-ci-e2e-test resourceVersion: "16993" uid: fa6dd1c6-b420-457e-a8d9-fcf42a82c881 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 0512425d-0b38-4e37-9c0c-0fe6fbf49e17 kind: Event lastTimestamp: "2026-07-31T15:01:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:01:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:36Z" name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6.18c767f9086d2d36 namespace: kserve-ci-e2e-test resourceVersion: "16995" uid: 13bea19c-7c2a-463f-9227-f2e5012b1f7f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T15:01:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 0512425d-0b38-4e37-9c0c-0fe6fbf49e17 kind: Event lastTimestamp: "2026-07-31T15:10:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:01:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:53Z" name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6.18c767f93ecb7f7b namespace: kserve-ci-e2e-test resourceVersion: "21305" uid: 58d3af46-8212-4c6c-9d21-27c8fca801ce reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 0512425d-0b38-4e37-9c0c-0fe6fbf49e17 kind: Event lastTimestamp: "2026-07-31T15:10:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:50Z" name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6.18c7687a165f7c8f namespace: kserve-ci-e2e-test resourceVersion: "21181" uid: 4d5cbd2f-9f0d-48ce-83d8-31db65be0204 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 0512425d-0b38-4e37-9c0c-0fe6fbf49e17 kind: Event lastTimestamp: "2026-07-31T15:10:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:10:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:50Z" name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6.18c7687a16606f6c namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: 0cf4793e-0078-41dc-9b53-03770ce202a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:10:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6 namespace: kserve-ci-e2e-test resourceVersion: "16917" uid: 0512425d-0b38-4e37-9c0c-0fe6fbf49e17 kind: Event lastTimestamp: "2026-07-31T15:10:53Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:10:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:53Z" name: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6.18c7687ab33d0725 namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: c6c55639-1dde-4173-bfd1-050f58068595 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a9565-predictor-5dbbcd748b namespace: kserve-ci-e2e-test resourceVersion: "16901" uid: bdab8620-12be-4d12-8dba-37f0a6a10817 kind: Event lastTimestamp: "2026-07-31T15:01:35Z" message: 'Created pod: success-200-isvc-a9565-predictor-5dbbcd748b-c7sp6' metadata: creationTimestamp: "2026-07-31T15:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:35Z" name: success-200-isvc-a9565-predictor-5dbbcd748b.18c767f8e58e673d namespace: kserve-ci-e2e-test resourceVersion: "16916" uid: 0b6d90b1-6c13-4b64-9131-b19ae8a11633 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a9565-predictor namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: 5bee3d29-2160-4287-aae0-24a61a3a4990 kind: Event lastTimestamp: "2026-07-31T15:01:35Z" message: Scaled up replica set success-200-isvc-a9565-predictor-5dbbcd748b from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:35Z" name: success-200-isvc-a9565-predictor.18c767f8e3b219e8 namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: d0c97308-2fa4-4ce2-896f-0ac79645af78 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:01:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9565-predictor namespace: kserve-ci-e2e-test resourceVersion: "16899" uid: f576f14e-2e1b-4809-ade0-30778441ffd0 kind: Event lastTimestamp: "2026-07-31T15:02:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:02:20Z" name: success-200-isvc-a9565-predictor.18c767fc62209425 namespace: kserve-ci-e2e-test resourceVersion: "17407" uid: 9c3837d4-e4a2-4ef2-9746-aadad508ccae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:01:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9565-predictor namespace: kserve-ci-e2e-test resourceVersion: "16899" uid: f576f14e-2e1b-4809-ade0-30778441ffd0 kind: Event lastTimestamp: "2026-07-31T15:02:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:02:21Z" name: success-200-isvc-a9565-predictor.18c767fc6285e590 namespace: kserve-ci-e2e-test resourceVersion: "17408" uid: 9891b831-a6a7-4e4f-87ea-dbd16ba48969 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T15:02:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9565-predictor namespace: kserve-ci-e2e-test resourceVersion: "17174" uid: f576f14e-2e1b-4809-ade0-30778441ffd0 kind: Event lastTimestamp: "2026-07-31T15:03:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:02:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:03:06Z" name: success-200-isvc-a9565-predictor.18c767ffe1a7a3cf namespace: kserve-ci-e2e-test resourceVersion: "17791" uid: a52660d0-86ca-4fe0-bb32-21c576ded30a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T15:02:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9565-predictor namespace: kserve-ci-e2e-test resourceVersion: "17174" uid: f576f14e-2e1b-4809-ade0-30778441ffd0 kind: Event lastTimestamp: "2026-07-31T15:03:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:02:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:03:06Z" name: success-200-isvc-a9565-predictor.18c767ffe21c2d04 namespace: kserve-ci-e2e-test resourceVersion: "17792" uid: a4fe0714-4c0e-4f70-ae48-498a4f9c5d2b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:01:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9565 namespace: kserve-ci-e2e-test resourceVersion: "16871" uid: 5c5d1649-bfb4-42e1-a091-211fd26ddada kind: Event lastTimestamp: "2026-07-31T15:02:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:01:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:02:23Z" name: success-200-isvc-a9565.18c767f8e3ab6fc0 namespace: kserve-ci-e2e-test resourceVersion: "17436" uid: dfb0355f-872e-45da-9205-861d65560afc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:02:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9565 namespace: kserve-ci-e2e-test resourceVersion: "17434" uid: 5c5d1649-bfb4-42e1-a091-211fd26ddada kind: Event lastTimestamp: "2026-07-31T15:02:23Z" message: InferenceService [success-200-isvc-a9565] is Ready metadata: creationTimestamp: "2026-07-31T15:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:02:23Z" name: success-200-isvc-a9565.18c76803fa1bfd3a namespace: kserve-ci-e2e-test resourceVersion: "17435" uid: 43569aac-a520-4bc3-945f-62cc61c52568 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:20:20.226970Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: d90a334f-7eea-41c6-aeaa-a44c41f481f3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv.18c768fea81a79c9 namespace: kserve-ci-e2e-test resourceVersion: "26190" uid: 9db8aecc-d9e9-4ab7-94e8-daf01a7734f4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: d90a334f-7eea-41c6-aeaa-a44c41f481f3 kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv.18c768fec1e12dd7 namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 481474a5-7a92-475e-8b6d-1448c6a38d31 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: d90a334f-7eea-41c6-aeaa-a44c41f481f3 kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv.18c768fec330cf8a namespace: kserve-ci-e2e-test resourceVersion: "26246" uid: ec71c6c5-709b-4c8c-b0da-68232cbf3eb5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: d90a334f-7eea-41c6-aeaa-a44c41f481f3 kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv.18c768fec64383a3 namespace: kserve-ci-e2e-test resourceVersion: "26249" uid: 8bc214a3-5018-4887-8b8c-40b74a5f3c2a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: d90a334f-7eea-41c6-aeaa-a44c41f481f3 kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv.18c768fec6bc5bc0 namespace: kserve-ci-e2e-test resourceVersion: "26250" uid: 66db529c-8a85-4e8c-b669-19e6897cbcbb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: d90a334f-7eea-41c6-aeaa-a44c41f481f3 kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv.18c768fec6c6c536 namespace: kserve-ci-e2e-test resourceVersion: "26251" uid: 6f89ad76-3a6e-473a-bed5-4e0d56bdaefb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: d90a334f-7eea-41c6-aeaa-a44c41f481f3 kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv.18c768fec95ace7a namespace: kserve-ci-e2e-test resourceVersion: "26252" uid: 917821db-827e-4e60-9849-e04b492cbeb3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: d90a334f-7eea-41c6-aeaa-a44c41f481f3 kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv.18c768fec9cf1cd0 namespace: kserve-ci-e2e-test resourceVersion: "26253" uid: d659bdb1-ff0a-4a5e-8395-952652688f7e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:20:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: d90a334f-7eea-41c6-aeaa-a44c41f481f3 kind: Event lastTimestamp: "2026-07-31T15:20:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:20:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:20:57Z" name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv.18c768ff015cda6f namespace: kserve-ci-e2e-test resourceVersion: "26555" uid: b0989040-a643-4a1c-bf9e-6b6897367198 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: d90a334f-7eea-41c6-aeaa-a44c41f481f3 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv.18c7690f244238d6 namespace: kserve-ci-e2e-test resourceVersion: "27069" uid: 48bf809b-39d3-4aad-80f0-542b4464e0b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: d90a334f-7eea-41c6-aeaa-a44c41f481f3 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv.18c7690f2442c6ca namespace: kserve-ci-e2e-test resourceVersion: "27072" uid: b749184d-ef15-4d54-a406-e96a5276659d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: d90a334f-7eea-41c6-aeaa-a44c41f481f3 kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv.18c7690f8987246f namespace: kserve-ci-e2e-test resourceVersion: "27135" uid: 7448b842-5c8c-4e1d-9895-9f139d905c8a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e31d5-predictor-7bf5c4d786 namespace: kserve-ci-e2e-test resourceVersion: "26161" uid: cd5931f4-f470-47e7-bcd4-5f3e83983b4a kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: 'Created pod: success-200-isvc-e31d5-predictor-7bf5c4d786-cwrxv' metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-e31d5-predictor-7bf5c4d786.18c768fea789ffd9 namespace: kserve-ci-e2e-test resourceVersion: "26183" uid: 077dfef9-126a-40c7-8f74-4e3ff46d5a64 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e31d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: 5c575382-e805-4c68-8b7f-7fd9d9383608 kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: Scaled up replica set success-200-isvc-e31d5-predictor-7bf5c4d786 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-e31d5-predictor.18c768fea35cddd3 namespace: kserve-ci-e2e-test resourceVersion: "26164" uid: be47669a-1624-409e-b53f-f8eeeb55c6e5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:20:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e31d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26160" uid: 76f77a49-e83f-4987-b6bc-c209637e5410 kind: Event lastTimestamp: "2026-07-31T15:21:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:05Z" name: success-200-isvc-e31d5-predictor.18c7690222349d5d namespace: kserve-ci-e2e-test resourceVersion: "26612" uid: 19f7ebc9-0627-43d3-9e00-7f5d21467c44 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:20:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e31d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26160" uid: 76f77a49-e83f-4987-b6bc-c209637e5410 kind: Event lastTimestamp: "2026-07-31T15:21:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:05Z" name: success-200-isvc-e31d5-predictor.18c769022296a54f namespace: kserve-ci-e2e-test resourceVersion: "26613" uid: 63de0d33-3c5f-45e5-8ee5-b5a8fbf581e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e31d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26384" uid: 76f77a49-e83f-4987-b6bc-c209637e5410 kind: Event lastTimestamp: "2026-07-31T15:21:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:20Z" name: success-200-isvc-e31d5-predictor.18c7690c9f204193 namespace: kserve-ci-e2e-test resourceVersion: "26733" uid: a45559e0-be61-4258-b9fb-88ee671c660c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e31d5-predictor namespace: kserve-ci-e2e-test resourceVersion: "26384" uid: 76f77a49-e83f-4987-b6bc-c209637e5410 kind: Event lastTimestamp: "2026-07-31T15:21:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:20Z" name: success-200-isvc-e31d5-predictor.18c7690c9f79df91 namespace: kserve-ci-e2e-test resourceVersion: "26734" uid: cb5f95e2-6fe8-4e8c-810c-ce2c3416ac79 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e31d5 namespace: kserve-ci-e2e-test resourceVersion: "26147" uid: 8807ffee-a66e-4240-bf76-16f70ac0d5a3 kind: Event lastTimestamp: "2026-07-31T15:21:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:21:07Z" name: success-200-isvc-e31d5.18c768fea3a4f0b6 namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 0af5a346-a563-445a-ae1e-a6e5a2223c5a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e31d5 namespace: kserve-ci-e2e-test resourceVersion: "26147" uid: 8807ffee-a66e-4240-bf76-16f70ac0d5a3 kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: 'Failed to update status for InferenceService "success-200-isvc-e31d5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e31d5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-e31d5.18c768fea48a364c namespace: kserve-ci-e2e-test resourceVersion: "26174" uid: d4c39aec-ddd7-4e70-84b7-5c8ef85e03c6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:20:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e31d5 namespace: kserve-ci-e2e-test resourceVersion: "26147" uid: 8807ffee-a66e-4240-bf76-16f70ac0d5a3 kind: Event lastTimestamp: "2026-07-31T15:20:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e31d5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T15:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:20:20Z" name: success-200-isvc-e31d5.18c768fea48aa9b8 namespace: kserve-ci-e2e-test resourceVersion: "26176" uid: 2074650c-2a27-499c-b0d7-0e27c949f489 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e31d5 namespace: kserve-ci-e2e-test resourceVersion: "26647" uid: 8807ffee-a66e-4240-bf76-16f70ac0d5a3 kind: Event lastTimestamp: "2026-07-31T15:21:07Z" message: InferenceService [success-200-isvc-e31d5] is Ready metadata: creationTimestamp: "2026-07-31T15:21:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:21:07Z" name: success-200-isvc-e31d5.18c76909bff9ce30 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: d473784c-ff63-4929-a40c-5a315adc6f16 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T14:59:05.628024Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e9535-predictor-7cdddc598d-krch6 to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T14:59:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T14:59:05Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767d5e415f093 namespace: kserve-ci-e2e-test resourceVersion: "15123" uid: b709116f-5634-42bb-b4f7-367d09e544bc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T14:59:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e9535-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T14:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:05Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767d5ef0fe58c namespace: kserve-ci-e2e-test resourceVersion: "15136" uid: f42de5bc-ffd6-4046-994c-e942e08ced35 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Add eth0 [10.134.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T14:59:06Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767d621f465a7 namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: 2f3e6fda-fcf2-496d-99e2-9fb6ba8f49cd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T14:59:06Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1833" metadata: creationTimestamp: "2026-07-31T14:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:06Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767d6246add41 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: 41566206-5a1e-4adb-83db-cffc5a4f8d06 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T14:59:22Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1833" in 15.757s (15.757s including waiting). Image size: 1334927461 bytes.' metadata: creationTimestamp: "2026-07-31T14:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:22Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767d9cfa65947 namespace: kserve-ci-e2e-test resourceVersion: "15445" uid: e8213ea1-028a-4173-afaa-f8ccd9181e3e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T14:59:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T14:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:22Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767d9d326cfbe namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: 046b2aa1-3d60-4283-ab5e-6d6ba228ebe0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T14:59:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T14:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:22Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767d9d39f1879 namespace: kserve-ci-e2e-test resourceVersion: "15448" uid: 074fb66a-c747-4943-86b2-16adacb5ca54 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T14:59:22Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-31T14:59:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:22Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767d9d3abed1c namespace: kserve-ci-e2e-test resourceVersion: "15449" uid: f765b9fd-2bb6-42a4-9f14-80df0abebaa5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T14:59:25Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.808s (2.808s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-31T14:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:25Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767da7b154add namespace: kserve-ci-e2e-test resourceVersion: "15474" uid: ea7b03de-f8e8-4cb1-8840-e1383b70b2ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T14:59:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T14:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:25Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767da7f03928d namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: b7669baf-22fe-4411-a974-9b9affd1c412 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T14:59:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T14:59:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T14:59:25Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767da7f7dc8c3 namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: 22229a7e-1d34-4432-91fe-d6bf62448aab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T14:59:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T15:00:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T14:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:42Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767dadcdb40e9 namespace: kserve-ci-e2e-test resourceVersion: "16334" uid: a8f76817-e258-4061-9c45-21e7d0879900 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767ebec02b26e namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: 7704734c-1b85-4e1e-aa36-fd0da2347c30 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T15:00:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:40Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767ebec032ed2 namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: da667bfe-4e01-4f14-873a-fcbb69a3133b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e9535-predictor-7cdddc598d-krch6 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: fe678310-682a-47b6-a15d-ec4e62263f98 kind: Event lastTimestamp: "2026-07-31T15:00:42Z" message: 'Readiness probe failed: Get "https://10.134.0.17:8643/healthz": dial tcp 10.134.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:00:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:42Z" name: success-200-isvc-e9535-predictor-7cdddc598d-krch6.18c767ec8ef85f8d namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: 306a9859-46f7-4071-9650-adbe30b81bbe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e9535-predictor-7cdddc598d namespace: kserve-ci-e2e-test resourceVersion: "15108" uid: 152fe393-2412-4c5e-80ca-52799ea73ab1 kind: Event lastTimestamp: "2026-07-31T14:59:05Z" message: 'Created pod: success-200-isvc-e9535-predictor-7cdddc598d-krch6' metadata: creationTimestamp: "2026-07-31T14:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T14:59:05Z" name: success-200-isvc-e9535-predictor-7cdddc598d.18c767d5e377d122 namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: ff00e3a2-d9f2-4425-8ef9-402a8ff01174 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e9535-predictor namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: a9f107ec-fedf-4b24-846b-0a8404130e17 kind: Event lastTimestamp: "2026-07-31T14:59:05Z" message: Scaled up replica set success-200-isvc-e9535-predictor-7cdddc598d from 0 to 1 metadata: creationTimestamp: "2026-07-31T14:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T14:59:05Z" name: success-200-isvc-e9535-predictor.18c767d5dcd08bc6 namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 049f152b-37d4-4456-b070-7967aa9ed5ee reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T14:59:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e9535-predictor namespace: kserve-ci-e2e-test resourceVersion: "15114" uid: 02ac16f3-8433-423e-b556-9f61c1b188bf kind: Event lastTimestamp: "2026-07-31T15:00:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T14:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:05Z" name: success-200-isvc-e9535-predictor.18c767d96245ce9e namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: b7fc134a-7e0e-4dca-9d2d-d754dd9f8a72 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-31T14:59:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e9535-predictor namespace: kserve-ci-e2e-test resourceVersion: "15114" uid: 02ac16f3-8433-423e-b556-9f61c1b188bf kind: Event lastTimestamp: "2026-07-31T15:00:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T14:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:05Z" name: success-200-isvc-e9535-predictor.18c767d962b95f76 namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: d95410e8-4af1-4d7e-81b8-820ec8fa94a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:00:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e9535-predictor namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 02ac16f3-8433-423e-b556-9f61c1b188bf kind: Event lastTimestamp: "2026-07-31T15:00:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:35Z" name: success-200-isvc-e9535-predictor.18c767e75f9f64e5 namespace: kserve-ci-e2e-test resourceVersion: "16118" uid: be290095-2713-47ba-9a8a-34b7ba11c1d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:00:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e9535-predictor namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 02ac16f3-8433-423e-b556-9f61c1b188bf kind: Event lastTimestamp: "2026-07-31T15:00:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:00:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:35Z" name: success-200-isvc-e9535-predictor.18c767e76010020f namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: 070c911f-2c63-4fe4-a1b0-34c20283f862 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T14:59:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e9535 namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 1f0e0103-ef9b-4a98-a213-641c51d9ef23 kind: Event lastTimestamp: "2026-07-31T15:00:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T14:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:13Z" name: success-200-isvc-e9535.18c767d5e2b41ea0 namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 007c6165-1d88-4647-80c1-0835aa98fca8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e9535 namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 1f0e0103-ef9b-4a98-a213-641c51d9ef23 kind: Event lastTimestamp: "2026-07-31T14:59:05Z" message: 'Failed to update status for InferenceService "success-200-isvc-e9535": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e9535": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T14:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T14:59:05Z" name: success-200-isvc-e9535.18c767d5e3313b73 namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: cfc834cf-fe03-48b7-a79d-a370b91bf131 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T14:59:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e9535 namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 1f0e0103-ef9b-4a98-a213-641c51d9ef23 kind: Event lastTimestamp: "2026-07-31T14:59:05Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e9535": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-31T14:59:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T14:59:05Z" name: success-200-isvc-e9535.18c767d5e331b2b0 namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: f657fa03-4475-4222-890b-05268129b377 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e9535 namespace: kserve-ci-e2e-test resourceVersion: "15904" uid: 1f0e0103-ef9b-4a98-a213-641c51d9ef23 kind: Event lastTimestamp: "2026-07-31T15:00:13Z" message: InferenceService [success-200-isvc-e9535] is Ready metadata: creationTimestamp: "2026-07-31T15:00:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:13Z" name: success-200-isvc-e9535.18c767e598367d07 namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: ae0add61-2751-49ee-a088-aa9ff676014d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:21:30.835601Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-faf8d-predictor-5896849644-7rtxg namespace: kserve-ci-e2e-test resourceVersion: "27005" uid: 5b3aad63-8ee8-4381-bffc-2af39bbccc34 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-faf8d-predictor-5896849644-7rtxg to ip-10-0-135-216.ec2.internal metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-faf8d-predictor-5896849644-7rtxg.18c7690f18b4ace3 namespace: kserve-ci-e2e-test resourceVersion: "27010" uid: ff687fba-2f70-41cb-bd1a-c6ca85234cb5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-faf8d-predictor-5896849644-7rtxg namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 5b3aad63-8ee8-4381-bffc-2af39bbccc34 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-faf8d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: success-200-isvc-faf8d-predictor-5896849644-7rtxg.18c7690f2759a33f namespace: kserve-ci-e2e-test resourceVersion: "27086" uid: bfc82476-cab5-4fe9-a533-e3304f1882e4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-faf8d-predictor-5896849644-7rtxg namespace: kserve-ci-e2e-test resourceVersion: "27015" uid: 5b3aad63-8ee8-4381-bffc-2af39bbccc34 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:21:31Z" name: success-200-isvc-faf8d-predictor-5896849644-7rtxg.18c7690f57103276 namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: fac0934c-d5a6-4164-8676-8591678b2742 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-faf8d-predictor-5896849644-7rtxg namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 5b3aad63-8ee8-4381-bffc-2af39bbccc34 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1833" already present on machine metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: success-200-isvc-faf8d-predictor-5896849644-7rtxg.18c7690f58e3788c namespace: kserve-ci-e2e-test resourceVersion: "27110" uid: 3719a57f-baac-4c0a-b72b-e5e032b43093 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-faf8d-predictor-5896849644-7rtxg namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 5b3aad63-8ee8-4381-bffc-2af39bbccc34 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: success-200-isvc-faf8d-predictor-5896849644-7rtxg.18c7690f5c3242ed namespace: kserve-ci-e2e-test resourceVersion: "27111" uid: 90b020af-5578-4dc3-90b7-85464bc24244 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-faf8d-predictor-5896849644-7rtxg namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 5b3aad63-8ee8-4381-bffc-2af39bbccc34 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: success-200-isvc-faf8d-predictor-5896849644-7rtxg.18c7690f5cb8e12a namespace: kserve-ci-e2e-test resourceVersion: "27112" uid: a9b2ae63-2370-4c10-a036-49a69e395d51 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-faf8d-predictor-5896849644-7rtxg namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 5b3aad63-8ee8-4381-bffc-2af39bbccc34 kind: Event lastTimestamp: "2026-07-31T15:21:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-31T15:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:31Z" name: success-200-isvc-faf8d-predictor-5896849644-7rtxg.18c7690f5cc5dc9c namespace: kserve-ci-e2e-test resourceVersion: "27114" uid: 2b916e31-688b-411a-88f4-3ba8adcffe69 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-faf8d-predictor-5896849644-7rtxg namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 5b3aad63-8ee8-4381-bffc-2af39bbccc34 kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: success-200-isvc-faf8d-predictor-5896849644-7rtxg.18c7690f60159772 namespace: kserve-ci-e2e-test resourceVersion: "27122" uid: 735b5f19-3200-4fc6-8d0f-77dc4a70b693 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-faf8d-predictor-5896849644-7rtxg namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 5b3aad63-8ee8-4381-bffc-2af39bbccc34 kind: Event lastTimestamp: "2026-07-31T15:21:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:21:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:21:32Z" name: success-200-isvc-faf8d-predictor-5896849644-7rtxg.18c7690f60948cea namespace: kserve-ci-e2e-test resourceVersion: "27123" uid: 642e696e-a48e-480b-97f1-2bdacad0e745 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-31T15:21:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-faf8d-predictor-5896849644-7rtxg namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 5b3aad63-8ee8-4381-bffc-2af39bbccc34 kind: Event lastTimestamp: "2026-07-31T15:22:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:21:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:22:11Z" name: success-200-isvc-faf8d-predictor-5896849644-7rtxg.18c7690fd6b03fb3 namespace: kserve-ci-e2e-test resourceVersion: "27487" uid: d65ee9bd-2f81-4649-80c1-5ca5c7995c2d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:30:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-faf8d-predictor-5896849644-7rtxg namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 5b3aad63-8ee8-4381-bffc-2af39bbccc34 kind: Event lastTimestamp: "2026-07-31T15:30:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-31T15:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:30:55Z" name: success-200-isvc-faf8d-predictor-5896849644-7rtxg.18c76992905366c6 namespace: kserve-ci-e2e-test resourceVersion: "31270" uid: c3167111-682d-4da1-8b09-d9f9f25731b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:30:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-faf8d-predictor-5896849644-7rtxg namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 5b3aad63-8ee8-4381-bffc-2af39bbccc34 kind: Event lastTimestamp: "2026-07-31T15:30:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-31T15:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:30:55Z" name: success-200-isvc-faf8d-predictor-5896849644-7rtxg.18c769929053ce8e namespace: kserve-ci-e2e-test resourceVersion: "31269" uid: 75a70d3f-42ff-4107-b02c-29a377376d31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-faf8d-predictor-5896849644-7rtxg namespace: kserve-ci-e2e-test resourceVersion: "27007" uid: 5b3aad63-8ee8-4381-bffc-2af39bbccc34 kind: Event lastTimestamp: "2026-07-31T15:30:56Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-31T15:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:30:56Z" name: success-200-isvc-faf8d-predictor-5896849644-7rtxg.18c76992b193c1b9 namespace: kserve-ci-e2e-test resourceVersion: "31294" uid: 5d3e7b4a-d0a4-44d2-b1a8-663689685741 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-216.ec2.internal source: component: kubelet host: ip-10-0-135-216.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-faf8d-predictor-5896849644 namespace: kserve-ci-e2e-test resourceVersion: "26997" uid: fedff0ee-8ae4-4c55-a6b3-47366013eb20 kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: 'Created pod: success-200-isvc-faf8d-predictor-5896849644-7rtxg' metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-faf8d-predictor-5896849644.18c7690f1841dba9 namespace: kserve-ci-e2e-test resourceVersion: "27006" uid: d36d84ca-f699-4a2e-8924-7fffd38927a5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-faf8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: a4b0d24f-b523-49d2-b967-cb585bbf80bf kind: Event lastTimestamp: "2026-07-31T15:21:30Z" message: Scaled up replica set success-200-isvc-faf8d-predictor-5896849644 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:21:30Z" name: success-200-isvc-faf8d-predictor.18c7690f1641297e namespace: kserve-ci-e2e-test resourceVersion: "26999" uid: 9c8b6ac7-5d1c-4c67-b2c0-8c4851d12f1f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:21:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-faf8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26996" uid: 5497d4fe-5fdd-4c31-8c7f-69c50f0c7c99 kind: Event lastTimestamp: "2026-07-31T15:22:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:21:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:00Z" name: success-200-isvc-faf8d-predictor.18c7691294cde31f namespace: kserve-ci-e2e-test resourceVersion: "27408" uid: b2e1a064-51b8-4c2b-ab0a-072fe03a495d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:21:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-faf8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26996" uid: 5497d4fe-5fdd-4c31-8c7f-69c50f0c7c99 kind: Event lastTimestamp: "2026-07-31T15:22:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:21:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:00Z" name: success-200-isvc-faf8d-predictor.18c76912953ed0a4 namespace: kserve-ci-e2e-test resourceVersion: "27409" uid: 7456acb0-1d33-416a-b34c-8ff674add8ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:22:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-faf8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "27273" uid: 5497d4fe-5fdd-4c31-8c7f-69c50f0c7c99 kind: Event lastTimestamp: "2026-07-31T15:22:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:45Z" name: success-200-isvc-faf8d-predictor.18c7691992d5b7c0 namespace: kserve-ci-e2e-test resourceVersion: "27863" uid: 17126344-6330-4418-ba78-bbf2bef9132e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-31T15:22:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-faf8d-predictor namespace: kserve-ci-e2e-test resourceVersion: "27273" uid: 5497d4fe-5fdd-4c31-8c7f-69c50f0c7c99 kind: Event lastTimestamp: "2026-07-31T15:22:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-31T15:22:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:45Z" name: success-200-isvc-faf8d-predictor.18c769199349205a namespace: kserve-ci-e2e-test resourceVersion: "27864" uid: 43dbceda-c7ed-42f9-a4f4-9ef74cdeee88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-31T15:21:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-faf8d namespace: kserve-ci-e2e-test resourceVersion: "26988" uid: bd4717dd-6f32-40c0-9c2e-d3b716024ccc kind: Event lastTimestamp: "2026-07-31T15:22:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-31T15:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:22:21Z" name: success-200-isvc-faf8d.18c7690f1664b64c namespace: kserve-ci-e2e-test resourceVersion: "27589" uid: fe42bdd6-2aff-431e-944b-0de97e7f25cd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-faf8d namespace: kserve-ci-e2e-test resourceVersion: "27587" uid: bd4717dd-6f32-40c0-9c2e-d3b716024ccc kind: Event lastTimestamp: "2026-07-31T15:22:21Z" message: InferenceService [success-200-isvc-faf8d] is Ready metadata: creationTimestamp: "2026-07-31T15:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:22:21Z" name: success-200-isvc-faf8d.18c7691acfd271d7 namespace: kserve-ci-e2e-test resourceVersion: "27588" uid: 7b5843ff-2263-412c-82b9-0f426781ed15 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:22:30.383917Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7902a-75876bd9fb-dtnk9 namespace: kserve-ci-e2e-test resourceVersion: "27679" uid: baa3fa5e-7cc6-4899-b142-98706019b01e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-7902a-75876bd9fb-dtnk9 to ip-10-0-136-197.ec2.internal metadata: creationTimestamp: "2026-07-31T15:22:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:22:30Z" name: switch-graph-7902a-75876bd9fb-dtnk9.18c7691cf60fdd3f namespace: kserve-ci-e2e-test resourceVersion: "27684" uid: 5c2471e5-1ef3-40b5-905d-54e83c6a59fb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:30Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7902a-75876bd9fb-dtnk9 namespace: kserve-ci-e2e-test resourceVersion: "27682" uid: baa3fa5e-7cc6-4899-b142-98706019b01e kind: Event lastTimestamp: "2026-07-31T15:22:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-7902a-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T15:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:22:30Z" name: switch-graph-7902a-75876bd9fb-dtnk9.18c7691cfde3191e namespace: kserve-ci-e2e-test resourceVersion: "27697" uid: bdb78ac3-bf89-405f-aa9e-2e3b09cd5507 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:31Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7902a-75876bd9fb-dtnk9 namespace: kserve-ci-e2e-test resourceVersion: "27690" uid: baa3fa5e-7cc6-4899-b142-98706019b01e kind: Event lastTimestamp: "2026-07-31T15:22:31Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:22:31Z" name: switch-graph-7902a-75876bd9fb-dtnk9.18c7691d33a9ff44 namespace: kserve-ci-e2e-test resourceVersion: "27712" uid: 3933950b-d2d4-46c1-8c0c-aeaa59d5d597 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7902a} kind: Pod name: switch-graph-7902a-75876bd9fb-dtnk9 namespace: kserve-ci-e2e-test resourceVersion: "27682" uid: baa3fa5e-7cc6-4899-b142-98706019b01e kind: Event lastTimestamp: "2026-07-31T15:22:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:45f424b4a00ee86756776b91e414e15d8d34d9dfbb27b1d7b22dfeb283636eda" already present on machine metadata: creationTimestamp: "2026-07-31T15:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:22:31Z" name: switch-graph-7902a-75876bd9fb-dtnk9.18c7691d3693fdb8 namespace: kserve-ci-e2e-test resourceVersion: "27714" uid: db71b92a-841d-42c7-81dc-9ff6ae69990d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7902a} kind: Pod name: switch-graph-7902a-75876bd9fb-dtnk9 namespace: kserve-ci-e2e-test resourceVersion: "27682" uid: baa3fa5e-7cc6-4899-b142-98706019b01e kind: Event lastTimestamp: "2026-07-31T15:22:31Z" message: 'Created container: switch-graph-7902a' metadata: creationTimestamp: "2026-07-31T15:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:22:31Z" name: switch-graph-7902a-75876bd9fb-dtnk9.18c7691d399f099f namespace: kserve-ci-e2e-test resourceVersion: "27716" uid: 85208bac-44a1-4807-855c-264c0aeed650 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7902a} kind: Pod name: switch-graph-7902a-75876bd9fb-dtnk9 namespace: kserve-ci-e2e-test resourceVersion: "27682" uid: baa3fa5e-7cc6-4899-b142-98706019b01e kind: Event lastTimestamp: "2026-07-31T15:22:31Z" message: Started container switch-graph-7902a metadata: creationTimestamp: "2026-07-31T15:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:22:31Z" name: switch-graph-7902a-75876bd9fb-dtnk9.18c7691d3a142452 namespace: kserve-ci-e2e-test resourceVersion: "27717" uid: 8eff29f3-a3e2-4359-b731-e8be436ba4c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7902a} kind: Pod name: switch-graph-7902a-75876bd9fb-dtnk9 namespace: kserve-ci-e2e-test resourceVersion: "27682" uid: baa3fa5e-7cc6-4899-b142-98706019b01e kind: Event lastTimestamp: "2026-07-31T15:38:49Z" message: Stopping container switch-graph-7902a metadata: creationTimestamp: "2026-07-31T15:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:38:49Z" name: switch-graph-7902a-75876bd9fb-dtnk9.18c76a00f715b11e namespace: kserve-ci-e2e-test resourceVersion: "34531" uid: 647f4eb7-07ce-475b-baba-1f9f73d0e972 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:38:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7902a} kind: Pod name: switch-graph-7902a-75876bd9fb-dtnk9 namespace: kserve-ci-e2e-test resourceVersion: "27682" uid: baa3fa5e-7cc6-4899-b142-98706019b01e kind: Event lastTimestamp: "2026-07-31T15:39:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-31T15:38:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:39:18Z" name: switch-graph-7902a-75876bd9fb-dtnk9.18c76a01da53d692 namespace: kserve-ci-e2e-test resourceVersion: "34816" uid: 54718120-a009-45f5-b399-82eacf595597 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-7902a-75876bd9fb namespace: kserve-ci-e2e-test resourceVersion: "27676" uid: 3a3387cc-4968-4b3b-ba3a-089bbac8c248 kind: Event lastTimestamp: "2026-07-31T15:22:30Z" message: 'Created pod: switch-graph-7902a-75876bd9fb-dtnk9' metadata: creationTimestamp: "2026-07-31T15:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:30Z" name: switch-graph-7902a-75876bd9fb.18c7691cf4f6f1e6 namespace: kserve-ci-e2e-test resourceVersion: "27683" uid: 8ae187bf-86c9-4977-bc6b-57c3b365e07e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-7902a namespace: kserve-ci-e2e-test resourceVersion: "27674" uid: 049b8c30-dccb-4e94-a724-b9843324daf3 kind: Event lastTimestamp: "2026-07-31T15:22:30Z" message: Scaled up replica set switch-graph-7902a-75876bd9fb from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:22:30Z" name: switch-graph-7902a.18c7691cf340207b namespace: kserve-ci-e2e-test resourceVersion: "27677" uid: 7779439c-6e9e-476f-b8b3-c42dd0eccbf2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:22:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-7902a namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: 1e286e10-192a-43cc-be6a-1d53458b62cb kind: Event lastTimestamp: "2026-07-31T15:22:30Z" message: InferenceGraph [switch-graph-7902a] is Ready metadata: creationTimestamp: "2026-07-31T15:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:22:30Z" name: switch-graph-7902a.18c7691cfb817d03 namespace: kserve-ci-e2e-test resourceVersion: "27695" uid: eda83cc4-145e-4687-a21f-c18b3b73df7f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:22:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7902a namespace: kserve-ci-e2e-test resourceVersion: "27675" uid: 84c06b54-a765-43d1-aeba-ae5aa795770b kind: Event lastTimestamp: "2026-07-31T15:23:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:23:00Z" name: switch-graph-7902a.18c7692072d9a19b namespace: kserve-ci-e2e-test resourceVersion: "27972" uid: 3d6d942f-6959-47df-b1ce-c01532e96dc8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:22:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7902a namespace: kserve-ci-e2e-test resourceVersion: "27675" uid: 84c06b54-a765-43d1-aeba-ae5aa795770b kind: Event lastTimestamp: "2026-07-31T15:23:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:23:00Z" name: switch-graph-7902a.18c769207353b215 namespace: kserve-ci-e2e-test resourceVersion: "27973" uid: 8b1b0776-d4d5-43b4-be45-9c86e8d7c2b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:01:40.217776Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7b0b5-5859dbd5c9-v6w2x namespace: kserve-ci-e2e-test resourceVersion: "17055" uid: 9c0fa4e2-4e21-4960-a822-32532eed1a5c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-7b0b5-5859dbd5c9-v6w2x to ip-10-0-136-197.ec2.internal metadata: creationTimestamp: "2026-07-31T15:01:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:01:40Z" name: switch-graph-7b0b5-5859dbd5c9-v6w2x.18c767f9e25a59fb namespace: kserve-ci-e2e-test resourceVersion: "17060" uid: 45216ed2-c450-4a23-ba2c-6b24f4e0286b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:40Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7b0b5-5859dbd5c9-v6w2x namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 9c0fa4e2-4e21-4960-a822-32532eed1a5c kind: Event lastTimestamp: "2026-07-31T15:01:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-7b0b5-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T15:01:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:40Z" name: switch-graph-7b0b5-5859dbd5c9-v6w2x.18c767f9ecc2b63d namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: 1f1fbc4f-0d9b-4881-b2e8-664cd4f8ed6a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:41Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-7b0b5-5859dbd5c9-v6w2x namespace: kserve-ci-e2e-test resourceVersion: "17065" uid: 9c0fa4e2-4e21-4960-a822-32532eed1a5c kind: Event lastTimestamp: "2026-07-31T15:01:41Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:01:41Z" name: switch-graph-7b0b5-5859dbd5c9-v6w2x.18c767fa1f9d16eb namespace: kserve-ci-e2e-test resourceVersion: "17087" uid: d2651c14-39f9-414a-8cb4-da840cf32225 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7b0b5} kind: Pod name: switch-graph-7b0b5-5859dbd5c9-v6w2x namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 9c0fa4e2-4e21-4960-a822-32532eed1a5c kind: Event lastTimestamp: "2026-07-31T15:01:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:45f424b4a00ee86756776b91e414e15d8d34d9dfbb27b1d7b22dfeb283636eda" already present on machine metadata: creationTimestamp: "2026-07-31T15:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:41Z" name: switch-graph-7b0b5-5859dbd5c9-v6w2x.18c767fa20eda103 namespace: kserve-ci-e2e-test resourceVersion: "17089" uid: ef2995b4-bae2-4704-9d80-fb5f47b29f78 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7b0b5} kind: Pod name: switch-graph-7b0b5-5859dbd5c9-v6w2x namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 9c0fa4e2-4e21-4960-a822-32532eed1a5c kind: Event lastTimestamp: "2026-07-31T15:01:41Z" message: 'Created container: switch-graph-7b0b5' metadata: creationTimestamp: "2026-07-31T15:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:41Z" name: switch-graph-7b0b5-5859dbd5c9-v6w2x.18c767fa244127e6 namespace: kserve-ci-e2e-test resourceVersion: "17090" uid: 8f3462d0-4c46-436b-ad7f-f82c927c979f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7b0b5} kind: Pod name: switch-graph-7b0b5-5859dbd5c9-v6w2x namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 9c0fa4e2-4e21-4960-a822-32532eed1a5c kind: Event lastTimestamp: "2026-07-31T15:01:41Z" message: Started container switch-graph-7b0b5 metadata: creationTimestamp: "2026-07-31T15:01:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:41Z" name: switch-graph-7b0b5-5859dbd5c9-v6w2x.18c767fa24bf410f namespace: kserve-ci-e2e-test resourceVersion: "17091" uid: d486866e-fa85-4515-b282-ea06ffe02760 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:09:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7b0b5} kind: Pod name: switch-graph-7b0b5-5859dbd5c9-v6w2x namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 9c0fa4e2-4e21-4960-a822-32532eed1a5c kind: Event lastTimestamp: "2026-07-31T15:09:54Z" message: Stopping container switch-graph-7b0b5 metadata: creationTimestamp: "2026-07-31T15:09:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:09:54Z" name: switch-graph-7b0b5-5859dbd5c9-v6w2x.18c7686d06d7a484 namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: 54f51003-96b4-4cb4-a06d-979f2ec06049 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:09:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-7b0b5} kind: Pod name: switch-graph-7b0b5-5859dbd5c9-v6w2x namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 9c0fa4e2-4e21-4960-a822-32532eed1a5c kind: Event lastTimestamp: "2026-07-31T15:10:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-31T15:09:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:10:22Z" name: switch-graph-7b0b5-5859dbd5c9-v6w2x.18c7686db8691387 namespace: kserve-ci-e2e-test resourceVersion: "20924" uid: bf6381da-45f0-490f-b8d4-7859aca12be8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-7b0b5-5859dbd5c9 namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: 252a587c-ff99-4d1a-8b6a-8e663a5f1540 kind: Event lastTimestamp: "2026-07-31T15:01:40Z" message: 'Created pod: switch-graph-7b0b5-5859dbd5c9-v6w2x' metadata: creationTimestamp: "2026-07-31T15:01:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:40Z" name: switch-graph-7b0b5-5859dbd5c9.18c767f9e1c64977 namespace: kserve-ci-e2e-test resourceVersion: "17056" uid: 433fcfce-b0fe-4467-9af2-0455a77e8c36 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-7b0b5 namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 0f967e65-35b1-44c1-b233-572ab7d9ad26 kind: Event lastTimestamp: "2026-07-31T15:01:40Z" message: Scaled up replica set switch-graph-7b0b5-5859dbd5c9 from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:01:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:01:40Z" name: switch-graph-7b0b5.18c767f9e0872131 namespace: kserve-ci-e2e-test resourceVersion: "17052" uid: 5e516876-109c-4f41-b536-e155dc9109c2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:01:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-7b0b5 namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 7990e3a0-a4ce-4562-9218-35bc2a7fd5dc kind: Event lastTimestamp: "2026-07-31T15:01:40Z" message: InferenceGraph [switch-graph-7b0b5] is Ready metadata: creationTimestamp: "2026-07-31T15:01:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:01:40Z" name: switch-graph-7b0b5.18c767f9e7acee5c namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: 3c22fabb-22c5-4128-8fc5-a50465242bbf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:01:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7b0b5 namespace: kserve-ci-e2e-test resourceVersion: "17049" uid: ac1f48a5-da3d-4993-a848-837a0226dc96 kind: Event lastTimestamp: "2026-07-31T15:02:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:01:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:02:10Z" name: switch-graph-7b0b5.18c767fd5f50fa62 namespace: kserve-ci-e2e-test resourceVersion: "17329" uid: 45920603-03a5-4b9c-8072-70f6ec6178b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:01:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-7b0b5 namespace: kserve-ci-e2e-test resourceVersion: "17049" uid: ac1f48a5-da3d-4993-a848-837a0226dc96 kind: Event lastTimestamp: "2026-07-31T15:02:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-31T15:01:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:02:10Z" name: switch-graph-7b0b5.18c767fd5fb8cc66 namespace: kserve-ci-e2e-test resourceVersion: "17330" uid: 54735ba5-9238-4a5b-b4af-84e4664dd6ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-31T15:00:25.703474Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e9535-68fb6f49bc-d8ks6 namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 48f34d81-40ba-45bd-ab27-02bbaad774ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-e9535-68fb6f49bc-d8ks6 to ip-10-0-136-197.ec2.internal metadata: creationTimestamp: "2026-07-31T15:00:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-31T15:00:25Z" name: switch-graph-e9535-68fb6f49bc-d8ks6.18c767e888f46a73 namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 9b42e1bf-2f88-4c3e-be1f-f5a21ceb223f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-755d7b5595-tc7m5 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-31T15:00:25Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e9535-68fb6f49bc-d8ks6 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: 48f34d81-40ba-45bd-ab27-02bbaad774ba kind: Event lastTimestamp: "2026-07-31T15:00:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-e9535-serving-cert" not found' metadata: creationTimestamp: "2026-07-31T15:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:26Z" name: switch-graph-e9535-68fb6f49bc-d8ks6.18c767e88f9f34e2 namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 854ae248-9fd3-495a-bbaa-22a1d0267fb5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:27Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e9535-68fb6f49bc-d8ks6 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 48f34d81-40ba-45bd-ab27-02bbaad774ba kind: Event lastTimestamp: "2026-07-31T15:00:27Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-31T15:00:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-31T15:00:27Z" name: switch-graph-e9535-68fb6f49bc-d8ks6.18c767e8fc50cf1c namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: 5858b693-8909-485e-b5cb-14556c525f73 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e9535} kind: Pod name: switch-graph-e9535-68fb6f49bc-d8ks6 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: 48f34d81-40ba-45bd-ab27-02bbaad774ba kind: Event lastTimestamp: "2026-07-31T15:00:27Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:45f424b4a00ee86756776b91e414e15d8d34d9dfbb27b1d7b22dfeb283636eda" metadata: creationTimestamp: "2026-07-31T15:00:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:27Z" name: switch-graph-e9535-68fb6f49bc-d8ks6.18c767e8fdbcbf9f namespace: kserve-ci-e2e-test resourceVersion: "16053" uid: d53443cf-a1b5-4733-9663-869c433442d7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e9535} kind: Pod name: switch-graph-e9535-68fb6f49bc-d8ks6 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: 48f34d81-40ba-45bd-ab27-02bbaad774ba kind: Event lastTimestamp: "2026-07-31T15:00:30Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:45f424b4a00ee86756776b91e414e15d8d34d9dfbb27b1d7b22dfeb283636eda" in 2.425s (2.425s including waiting). Image size: 218226790 bytes.' metadata: creationTimestamp: "2026-07-31T15:00:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:30Z" name: switch-graph-e9535-68fb6f49bc-d8ks6.18c767e98e55950c namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 09403f5c-8bcb-417c-b397-a7b1c77982c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e9535} kind: Pod name: switch-graph-e9535-68fb6f49bc-d8ks6 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: 48f34d81-40ba-45bd-ab27-02bbaad774ba kind: Event lastTimestamp: "2026-07-31T15:00:30Z" message: 'Created container: switch-graph-e9535' metadata: creationTimestamp: "2026-07-31T15:00:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:30Z" name: switch-graph-e9535-68fb6f49bc-d8ks6.18c767e99136ff3e namespace: kserve-ci-e2e-test resourceVersion: "16076" uid: 87be3ef2-ad10-4578-99f8-b08f683050e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e9535} kind: Pod name: switch-graph-e9535-68fb6f49bc-d8ks6 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: 48f34d81-40ba-45bd-ab27-02bbaad774ba kind: Event lastTimestamp: "2026-07-31T15:00:30Z" message: Started container switch-graph-e9535 metadata: creationTimestamp: "2026-07-31T15:00:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:30Z" name: switch-graph-e9535-68fb6f49bc-d8ks6.18c767e991ad6e5b namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: df62e361-1506-4c77-840c-9651195f4233 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e9535} kind: Pod name: switch-graph-e9535-68fb6f49bc-d8ks6 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: 48f34d81-40ba-45bd-ab27-02bbaad774ba kind: Event lastTimestamp: "2026-07-31T15:00:39Z" message: Stopping container switch-graph-e9535 metadata: creationTimestamp: "2026-07-31T15:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:00:39Z" name: switch-graph-e9535-68fb6f49bc-d8ks6.18c767ebd963e95c namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 642c14fa-b7a0-4f04-a97b-887f48332c46 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-31T15:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e9535} kind: Pod name: switch-graph-e9535-68fb6f49bc-d8ks6 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: 48f34d81-40ba-45bd-ab27-02bbaad774ba kind: Event lastTimestamp: "2026-07-31T15:01:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-31T15:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-31T15:01:06Z" name: switch-graph-e9535-68fb6f49bc-d8ks6.18c767ec30932e25 namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 5243cd6c-ba78-488a-8496-44c62e2f18b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-197.ec2.internal source: component: kubelet host: ip-10-0-136-197.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-e9535-68fb6f49bc namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 3da4fd40-6250-42ba-98fb-46b8b720064c kind: Event lastTimestamp: "2026-07-31T15:00:25Z" message: 'Created pod: switch-graph-e9535-68fb6f49bc-d8ks6' metadata: creationTimestamp: "2026-07-31T15:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:25Z" name: switch-graph-e9535-68fb6f49bc.18c767e8887eabb3 namespace: kserve-ci-e2e-test resourceVersion: "16023" uid: 58038d92-79a5-4a11-bbb8-7654df2ce27c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-e9535 namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 737338f3-e808-4e89-8e00-ffc832edd886 kind: Event lastTimestamp: "2026-07-31T15:00:25Z" message: Scaled up replica set switch-graph-e9535-68fb6f49bc from 0 to 1 metadata: creationTimestamp: "2026-07-31T15:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-31T15:00:25Z" name: switch-graph-e9535.18c767e886e47bd3 namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: 5eaf87ab-3ff5-424f-8f7a-6fcf0c24dcaf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-31T15:00:25Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-e9535 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 7c536aec-9364-47f5-b121-c2331dccae23 kind: Event lastTimestamp: "2026-07-31T15:00:25Z" message: InferenceGraph [switch-graph-e9535] is Ready metadata: creationTimestamp: "2026-07-31T15:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-31T15:00:25Z" name: switch-graph-e9535.18c767e88e53e0c4 namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 535a3e12-d347-43be-83c0-f82ef88eabd7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "34820"