--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:06:40.909095Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7b7ab-9fcfc967-k4fg6 namespace: kserve-ci-e2e-test resourceVersion: "21585" uid: 6e41de04-13ec-487c-8f26-eb5601f5b713 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-7b7ab-9fcfc967-k4fg6 to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:06:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:06:40Z" name: ensemble-graph-7b7ab-9fcfc967-k4fg6.18be7e253e483b5e namespace: kserve-ci-e2e-test resourceVersion: "21590" uid: f87d5a9a-d21a-4e90-a36b-367a2dd49ff5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:41Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7b7ab-9fcfc967-k4fg6 namespace: kserve-ci-e2e-test resourceVersion: "21587" uid: 6e41de04-13ec-487c-8f26-eb5601f5b713 kind: Event lastTimestamp: "2026-07-02T14:06:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-7b7ab-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:41Z" name: ensemble-graph-7b7ab-9fcfc967-k4fg6.18be7e2548f23206 namespace: kserve-ci-e2e-test resourceVersion: "21603" uid: e8e9defb-7e2e-4e63-8ea3-dbb2c18804ac reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:41Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7b7ab-9fcfc967-k4fg6 namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: 6e41de04-13ec-487c-8f26-eb5601f5b713 kind: Event lastTimestamp: "2026-07-02T14:06:41Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:06:41Z" name: ensemble-graph-7b7ab-9fcfc967-k4fg6.18be7e257b95f5dd namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 4ac498f1-555d-4c2f-819a-fca2fa0c93aa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7b7ab} kind: Pod name: ensemble-graph-7b7ab-9fcfc967-k4fg6 namespace: kserve-ci-e2e-test resourceVersion: "21587" uid: 6e41de04-13ec-487c-8f26-eb5601f5b713 kind: Event lastTimestamp: "2026-07-02T14:06:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" already present on machine metadata: creationTimestamp: "2026-07-02T14:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:41Z" name: ensemble-graph-7b7ab-9fcfc967-k4fg6.18be7e257cd3beb3 namespace: kserve-ci-e2e-test resourceVersion: "21616" uid: 4bc371ed-2275-49b1-a9f9-fcc9ae42542d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7b7ab} kind: Pod name: ensemble-graph-7b7ab-9fcfc967-k4fg6 namespace: kserve-ci-e2e-test resourceVersion: "21587" uid: 6e41de04-13ec-487c-8f26-eb5601f5b713 kind: Event lastTimestamp: "2026-07-02T14:06:42Z" message: 'Created container: ensemble-graph-7b7ab' metadata: creationTimestamp: "2026-07-02T14:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:42Z" name: ensemble-graph-7b7ab-9fcfc967-k4fg6.18be7e257f8de69d namespace: kserve-ci-e2e-test resourceVersion: "21618" uid: 2d875f3f-585c-4311-8277-2a6b18e55baa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7b7ab} kind: Pod name: ensemble-graph-7b7ab-9fcfc967-k4fg6 namespace: kserve-ci-e2e-test resourceVersion: "21587" uid: 6e41de04-13ec-487c-8f26-eb5601f5b713 kind: Event lastTimestamp: "2026-07-02T14:06:42Z" message: Started container ensemble-graph-7b7ab metadata: creationTimestamp: "2026-07-02T14:06:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:42Z" name: ensemble-graph-7b7ab-9fcfc967-k4fg6.18be7e257ffde3ca namespace: kserve-ci-e2e-test resourceVersion: "21619" uid: b1a5f4d0-e1c6-48f0-81ff-a1cb3297d5a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7b7ab} kind: Pod name: ensemble-graph-7b7ab-9fcfc967-k4fg6 namespace: kserve-ci-e2e-test resourceVersion: "21587" uid: 6e41de04-13ec-487c-8f26-eb5601f5b713 kind: Event lastTimestamp: "2026-07-02T14:14:55Z" message: Stopping container ensemble-graph-7b7ab metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:55Z" name: ensemble-graph-7b7ab-9fcfc967-k4fg6.18be7e9870b56d71 namespace: kserve-ci-e2e-test resourceVersion: "25101" uid: dfd8888a-3488-4a4c-9787-b303a4361a01 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7b7ab} kind: Pod name: ensemble-graph-7b7ab-9fcfc967-k4fg6 namespace: kserve-ci-e2e-test resourceVersion: "21587" uid: 6e41de04-13ec-487c-8f26-eb5601f5b713 kind: Event lastTimestamp: "2026-07-02T14:15:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:23Z" name: ensemble-graph-7b7ab-9fcfc967-k4fg6.18be7e990ae27b1b namespace: kserve-ci-e2e-test resourceVersion: "25698" uid: 4019e320-05d5-4c28-8c9b-692f13f2f0d8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-7b7ab-9fcfc967 namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: 51730177-e19f-4c79-8f25-a5d3e775a237 kind: Event lastTimestamp: "2026-07-02T14:06:40Z" message: 'Created pod: ensemble-graph-7b7ab-9fcfc967-k4fg6' metadata: creationTimestamp: "2026-07-02T14:06:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:40Z" name: ensemble-graph-7b7ab-9fcfc967.18be7e253ddb13a1 namespace: kserve-ci-e2e-test resourceVersion: "21586" uid: 220797ad-192a-4361-bac3-d679c577b911 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-7b7ab namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: 289d61ee-c4dd-40dc-a561-2fae875cc8ba kind: Event lastTimestamp: "2026-07-02T14:06:40Z" message: Scaled up replica set ensemble-graph-7b7ab-9fcfc967 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:06:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:40Z" name: ensemble-graph-7b7ab.18be7e253d2bc3f8 namespace: kserve-ci-e2e-test resourceVersion: "21583" uid: 2a178519-4bcd-453a-88f9-1fb02ae3790c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-7b7ab namespace: kserve-ci-e2e-test resourceVersion: "21598" uid: 7c6748fc-464f-466a-8657-0493ac1e3a25 kind: Event lastTimestamp: "2026-07-02T14:06:40Z" message: InferenceGraph [ensemble-graph-7b7ab] is Ready metadata: creationTimestamp: "2026-07-02T14:06:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:40Z" name: ensemble-graph-7b7ab.18be7e2542da46af namespace: kserve-ci-e2e-test resourceVersion: "21600" uid: c9c9003f-8af4-4fd1-a7e9-9ca4c79766fa reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-7b7ab namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: c3bf7f4c-e97a-4a23-a012-62be2becdda4 kind: Event lastTimestamp: "2026-07-02T14:06:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:55Z" name: ensemble-graph-7b7ab.18be7e28bba1719c namespace: kserve-ci-e2e-test resourceVersion: "21735" uid: 76a248cd-39da-48ef-b6d1-169171b8195b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-7b7ab namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: c3bf7f4c-e97a-4a23-a012-62be2becdda4 kind: Event lastTimestamp: "2026-07-02T14:06:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:06:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:55Z" name: ensemble-graph-7b7ab.18be7e28bbda8c37 namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 71911f41-4d52-4aeb-ae6e-a3613dae0dea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-7b7ab namespace: kserve-ci-e2e-test resourceVersion: "21733" uid: c3bf7f4c-e97a-4a23-a012-62be2becdda4 kind: Event lastTimestamp: "2026-07-02T14:07:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:10Z" name: ensemble-graph-7b7ab.18be7e2c3ac21ac6 namespace: kserve-ci-e2e-test resourceVersion: "21851" uid: a4b55122-59fd-434e-850e-1c694860837c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-7b7ab namespace: kserve-ci-e2e-test resourceVersion: "21733" uid: c3bf7f4c-e97a-4a23-a012-62be2becdda4 kind: Event lastTimestamp: "2026-07-02T14:07:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:07:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:10Z" name: ensemble-graph-7b7ab.18be7e2c3b2b29c7 namespace: kserve-ci-e2e-test resourceVersion: "21853" uid: a4189e7e-7f57-4230-97e7-f52806e2fbb4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:05:30.616854Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c675b-768bbd884f-vxx9q namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: 816c127e-16bd-4132-881c-3199fe1e0907 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-c675b-768bbd884f-vxx9q to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:05:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:05:30Z" name: ensemble-graph-c675b-768bbd884f-vxx9q.18be7e14e08a78cf namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 8d668acc-0e7c-4977-b27f-29d5a10fe5be reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:31Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c675b-768bbd884f-vxx9q namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 816c127e-16bd-4132-881c-3199fe1e0907 kind: Event lastTimestamp: "2026-07-02T14:05:31Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:05:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:05:31Z" name: ensemble-graph-c675b-768bbd884f-vxx9q.18be7e14fb582293 namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: 6d36348d-d105-4097-80d4-3279a9ea1754 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c675b} kind: Pod name: ensemble-graph-c675b-768bbd884f-vxx9q namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 816c127e-16bd-4132-881c-3199fe1e0907 kind: Event lastTimestamp: "2026-07-02T14:05:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" already present on machine metadata: creationTimestamp: "2026-07-02T14:05:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:31Z" name: ensemble-graph-c675b-768bbd884f-vxx9q.18be7e14fcc118ee namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 92a63fd6-08fb-4146-a558-8f97dcbd3af7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c675b} kind: Pod name: ensemble-graph-c675b-768bbd884f-vxx9q namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 816c127e-16bd-4132-881c-3199fe1e0907 kind: Event lastTimestamp: "2026-07-02T14:05:31Z" message: 'Created container: ensemble-graph-c675b' metadata: creationTimestamp: "2026-07-02T14:05:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:31Z" name: ensemble-graph-c675b-768bbd884f-vxx9q.18be7e14ffac0899 namespace: kserve-ci-e2e-test resourceVersion: "20645" uid: 9c90d885-731d-4907-8f3b-d6ae4b427ea3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c675b} kind: Pod name: ensemble-graph-c675b-768bbd884f-vxx9q namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 816c127e-16bd-4132-881c-3199fe1e0907 kind: Event lastTimestamp: "2026-07-02T14:05:31Z" message: Started container ensemble-graph-c675b metadata: creationTimestamp: "2026-07-02T14:05:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:31Z" name: ensemble-graph-c675b-768bbd884f-vxx9q.18be7e15001c27a2 namespace: kserve-ci-e2e-test resourceVersion: "20647" uid: 422e2d5b-4e9f-477f-b0df-8e8bc36478d1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c675b} kind: Pod name: ensemble-graph-c675b-768bbd884f-vxx9q namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 816c127e-16bd-4132-881c-3199fe1e0907 kind: Event lastTimestamp: "2026-07-02T14:05:40Z" message: Stopping container ensemble-graph-c675b metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:40Z" name: ensemble-graph-c675b-768bbd884f-vxx9q.18be7e173a1a8d00 namespace: kserve-ci-e2e-test resourceVersion: "20740" uid: de6f7311-da0c-4c79-b875-438a3ead8977 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c675b} kind: Pod name: ensemble-graph-c675b-768bbd884f-vxx9q namespace: kserve-ci-e2e-test resourceVersion: "20617" uid: 816c127e-16bd-4132-881c-3199fe1e0907 kind: Event lastTimestamp: "2026-07-02T14:06:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:07Z" name: ensemble-graph-c675b-768bbd884f-vxx9q.18be7e17bdc388ac namespace: kserve-ci-e2e-test resourceVersion: "21096" uid: bcf5b9ff-408a-43cd-a22b-bbc41f3e6631 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-c675b-768bbd884f namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 4cbf74d8-c067-4779-9f98-778008c15986 kind: Event lastTimestamp: "2026-07-02T14:05:30Z" message: 'Created pod: ensemble-graph-c675b-768bbd884f-vxx9q' metadata: creationTimestamp: "2026-07-02T14:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:30Z" name: ensemble-graph-c675b-768bbd884f.18be7e14dfef025f namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: cba20083-db42-4aff-9b34-c0f4c10c9055 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-c675b namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: 5d2ca344-4432-45ff-9564-4924a44bdfb0 kind: Event lastTimestamp: "2026-07-02T14:05:30Z" message: Scaled up replica set ensemble-graph-c675b-768bbd884f from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:30Z" name: ensemble-graph-c675b.18be7e14df33d9be namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: dad7d690-5d57-442d-86d7-7246e72a3323 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c675b namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: 5f4917b1-a31b-4c73-9b40-ef86587fdb3a kind: Event lastTimestamp: "2026-07-02T14:05:30Z" message: InferenceGraph [ensemble-graph-c675b] is Ready metadata: creationTimestamp: "2026-07-02T14:05:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:30Z" name: ensemble-graph-c675b.18be7e14e50821c6 namespace: kserve-ci-e2e-test resourceVersion: "20630" uid: 4ab7d5c7-3409-4542-8541-c32130a89d98 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:14:55.956151Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-09121-predictor-78d4984875-8wwtg namespace: kserve-ci-e2e-test resourceVersion: "25166" uid: 3506c5ec-7374-43af-a888-c0e6eb72dd6d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-09121-predictor-78d4984875-8wwtg to ip-10-0-133-209.ec2.internal metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:14:55Z" name: error-404-isvc-09121-predictor-78d4984875-8wwtg.18be7e9881631a2f namespace: kserve-ci-e2e-test resourceVersion: "25172" uid: 2969901f-78fe-4bba-8dd7-be5d42659d64 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-09121-predictor-78d4984875-8wwtg namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 3506c5ec-7374-43af-a888-c0e6eb72dd6d kind: Event lastTimestamp: "2026-07-02T14:14:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-09121-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:56Z" name: error-404-isvc-09121-predictor-78d4984875-8wwtg.18be7e988d441181 namespace: kserve-ci-e2e-test resourceVersion: "25199" uid: bf797ee1-cc3c-46ee-9dbb-17356180a387 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-09121-predictor-78d4984875-8wwtg namespace: kserve-ci-e2e-test resourceVersion: "25185" uid: 3506c5ec-7374-43af-a888-c0e6eb72dd6d kind: Event lastTimestamp: "2026-07-02T14:14:56Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:14:56Z" name: error-404-isvc-09121-predictor-78d4984875-8wwtg.18be7e98bef58db3 namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: 6bb80bf8-beb5-49aa-8384-79c9f6e3f3b9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-09121-predictor-78d4984875-8wwtg namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 3506c5ec-7374-43af-a888-c0e6eb72dd6d kind: Event lastTimestamp: "2026-07-02T14:14:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:57Z" name: error-404-isvc-09121-predictor-78d4984875-8wwtg.18be7e98c06756a3 namespace: kserve-ci-e2e-test resourceVersion: "25220" uid: 5b7cd8fe-f6f8-46c8-9223-afb97b95368f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-09121-predictor-78d4984875-8wwtg namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 3506c5ec-7374-43af-a888-c0e6eb72dd6d kind: Event lastTimestamp: "2026-07-02T14:14:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:57Z" name: error-404-isvc-09121-predictor-78d4984875-8wwtg.18be7e98c3b814be namespace: kserve-ci-e2e-test resourceVersion: "25221" uid: a3be2fea-bea6-486c-a6df-0165fca53edb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-09121-predictor-78d4984875-8wwtg namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 3506c5ec-7374-43af-a888-c0e6eb72dd6d kind: Event lastTimestamp: "2026-07-02T14:14:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:57Z" name: error-404-isvc-09121-predictor-78d4984875-8wwtg.18be7e98c434bfc2 namespace: kserve-ci-e2e-test resourceVersion: "25222" uid: 05ed2dd2-bbc0-4ba2-bc1a-4c0d67c61786 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-09121-predictor-78d4984875-8wwtg namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 3506c5ec-7374-43af-a888-c0e6eb72dd6d kind: Event lastTimestamp: "2026-07-02T14:14:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:57Z" name: error-404-isvc-09121-predictor-78d4984875-8wwtg.18be7e98c43f4051 namespace: kserve-ci-e2e-test resourceVersion: "25223" uid: 21a18ab9-3e16-4b8d-a934-2268559fb753 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-09121-predictor-78d4984875-8wwtg namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 3506c5ec-7374-43af-a888-c0e6eb72dd6d kind: Event lastTimestamp: "2026-07-02T14:14:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:57Z" name: error-404-isvc-09121-predictor-78d4984875-8wwtg.18be7e98c77b8d69 namespace: kserve-ci-e2e-test resourceVersion: "25224" uid: 0f9ee52e-7ecb-4c7b-a65d-0d564bd83470 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-09121-predictor-78d4984875-8wwtg namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 3506c5ec-7374-43af-a888-c0e6eb72dd6d kind: Event lastTimestamp: "2026-07-02T14:14:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:57Z" name: error-404-isvc-09121-predictor-78d4984875-8wwtg.18be7e98c7f75667 namespace: kserve-ci-e2e-test resourceVersion: "25226" uid: b5d34aab-86bc-49b6-8d71-552659e061ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:14:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-09121-predictor-78d4984875-8wwtg namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 3506c5ec-7374-43af-a888-c0e6eb72dd6d kind: Event lastTimestamp: "2026-07-02T14:15:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:14:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:33Z" name: error-404-isvc-09121-predictor-78d4984875-8wwtg.18be7e98ce0186b4 namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: a476d978-9434-42a2-ad9f-d4770283fc98 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-09121-predictor-78d4984875-8wwtg namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 3506c5ec-7374-43af-a888-c0e6eb72dd6d kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-09121-predictor-78d4984875-8wwtg.18be7ea8e2e38947 namespace: kserve-ci-e2e-test resourceVersion: "26358" uid: cbaf744b-3f5c-4479-86cc-0f2dee45b4c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-09121-predictor-78d4984875-8wwtg namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 3506c5ec-7374-43af-a888-c0e6eb72dd6d kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-09121-predictor-78d4984875-8wwtg.18be7ea8e2e3f79e namespace: kserve-ci-e2e-test resourceVersion: "26359" uid: abaf0c4b-7204-4b86-abbd-f4109bf553be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-09121-predictor-78d4984875-8wwtg namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 3506c5ec-7374-43af-a888-c0e6eb72dd6d kind: Event lastTimestamp: "2026-07-02T14:16:08Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:16:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:08Z" name: error-404-isvc-09121-predictor-78d4984875-8wwtg.18be7ea956352644 namespace: kserve-ci-e2e-test resourceVersion: "26414" uid: 13722e33-ef69-4164-a8df-580325c82162 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-09121-predictor-78d4984875 namespace: kserve-ci-e2e-test resourceVersion: "25158" uid: 977992f9-ef01-447d-bfb8-4f9285164632 kind: Event lastTimestamp: "2026-07-02T14:14:55Z" message: 'Created pod: error-404-isvc-09121-predictor-78d4984875-8wwtg' metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:14:55Z" name: error-404-isvc-09121-predictor-78d4984875.18be7e9880f055f9 namespace: kserve-ci-e2e-test resourceVersion: "25169" uid: 6745feee-4347-4123-8224-b0bf900d1190 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-09121-predictor namespace: kserve-ci-e2e-test resourceVersion: "25153" uid: 53b563c2-30d9-4cc5-9b49-da4c8ae12c0e kind: Event lastTimestamp: "2026-07-02T14:14:55Z" message: Scaled up replica set error-404-isvc-09121-predictor-78d4984875 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:14:55Z" name: error-404-isvc-09121-predictor.18be7e987f0d36c3 namespace: kserve-ci-e2e-test resourceVersion: "25159" uid: 247f25f1-0c38-40af-998c-643ca2f32b69 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:15:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-09121-predictor namespace: kserve-ci-e2e-test resourceVersion: "25156" uid: 60e9f403-bc54-43ec-8d08-cb473d2562a2 kind: Event lastTimestamp: "2026-07-02T14:15:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:25Z" name: error-404-isvc-09121-predictor.18be7e9bfdd9b244 namespace: kserve-ci-e2e-test resourceVersion: "25723" uid: 3f97c756-af2d-46ab-8c4a-c174bc371836 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:15:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-09121-predictor namespace: kserve-ci-e2e-test resourceVersion: "25156" uid: 60e9f403-bc54-43ec-8d08-cb473d2562a2 kind: Event lastTimestamp: "2026-07-02T14:15:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:25Z" name: error-404-isvc-09121-predictor.18be7e9bfe4ca4f8 namespace: kserve-ci-e2e-test resourceVersion: "25724" uid: 1daffff7-53c4-45da-8c60-69be9e9e39a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:15:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-09121-predictor namespace: kserve-ci-e2e-test resourceVersion: "25346" uid: 60e9f403-bc54-43ec-8d08-cb473d2562a2 kind: Event lastTimestamp: "2026-07-02T14:15:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:55Z" name: error-404-isvc-09121-predictor.18be7ea2fbcd9687 namespace: kserve-ci-e2e-test resourceVersion: "26160" uid: 10cd7f78-6707-4256-a5bb-254080f7f0bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:15:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-09121-predictor namespace: kserve-ci-e2e-test resourceVersion: "25346" uid: 60e9f403-bc54-43ec-8d08-cb473d2562a2 kind: Event lastTimestamp: "2026-07-02T14:15:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:55Z" name: error-404-isvc-09121-predictor.18be7ea2fc28361f namespace: kserve-ci-e2e-test resourceVersion: "26162" uid: 4cc855cb-57d3-4519-b525-a98fc79755e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:14:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-09121 namespace: kserve-ci-e2e-test resourceVersion: "25123" uid: 7a0d67fe-aa7e-448d-a6be-e1453b1adbb1 kind: Event lastTimestamp: "2026-07-02T14:15:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:15:43Z" name: error-404-isvc-09121.18be7e987f5c4494 namespace: kserve-ci-e2e-test resourceVersion: "25887" uid: bb05f5f5-97e1-421d-b8fd-ca9119cb4723 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-09121 namespace: kserve-ci-e2e-test resourceVersion: "25123" uid: 7a0d67fe-aa7e-448d-a6be-e1453b1adbb1 kind: Event lastTimestamp: "2026-07-02T14:14:55Z" message: 'Failed to update status for InferenceService "error-404-isvc-09121": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-09121": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:14:55Z" name: error-404-isvc-09121.18be7e987fc9bfe0 namespace: kserve-ci-e2e-test resourceVersion: "25163" uid: 7f9dc12f-7fb2-4a44-be35-c8a542c9a388 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-09121 namespace: kserve-ci-e2e-test resourceVersion: "25123" uid: 7a0d67fe-aa7e-448d-a6be-e1453b1adbb1 kind: Event lastTimestamp: "2026-07-02T14:14:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-09121": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:14:55Z" name: error-404-isvc-09121.18be7e987fc9f452 namespace: kserve-ci-e2e-test resourceVersion: "25164" uid: 648265d2-6814-4e8a-a82c-06745ddce0d4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-09121 namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: 7a0d67fe-aa7e-448d-a6be-e1453b1adbb1 kind: Event lastTimestamp: "2026-07-02T14:15:43Z" message: InferenceService [error-404-isvc-09121] is Ready metadata: creationTimestamp: "2026-07-02T14:15:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:15:43Z" name: error-404-isvc-09121.18be7ea38a5e476f namespace: kserve-ci-e2e-test resourceVersion: "25885" uid: 448c4034-dffe-4d8b-82b2-c5fb084b1af0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:53:41.789588Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14469" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-40523-predictor-57f4887bdf-rkfb5 to ip-10-0-133-209.ec2.internal metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:53:41Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d6fd723f8e2 namespace: kserve-ci-e2e-test resourceVersion: "14475" uid: 3a7d22b8-40b8-49f5-837a-12f308b47e6a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14481" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: "2026-07-02T13:53:42Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:53:42Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d6ff0b3e7aa namespace: kserve-ci-e2e-test resourceVersion: "14540" uid: 4190cf91-c784-4424-8968-402db3b2c9bf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: "2026-07-02T13:53:42Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1695" metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:42Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d6ff2730124 namespace: kserve-ci-e2e-test resourceVersion: "14546" uid: fcf5e52e-d7b2-4a56-9245-ee624133156f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: "2026-07-02T13:53:54Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1695" in 12.295s (12.295s including waiting). Image size: 1334585467 bytes.' metadata: creationTimestamp: "2026-07-02T13:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:54Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d72cf51bca4 namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 2ecc8ba5-6ebc-4b00-b5de-553924548828 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: "2026-07-02T13:53:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:54Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d72d2d256ef namespace: kserve-ci-e2e-test resourceVersion: "14689" uid: 28fa9f89-5363-492a-acb0-e4f287a02caf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: "2026-07-02T13:53:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:54Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d72d34a102d namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: 6eab4514-63a1-4ebe-ac0b-0f4559407db9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: "2026-07-02T13:53:54Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T13:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:54Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d72d356a6b4 namespace: kserve-ci-e2e-test resourceVersion: "14691" uid: e38d1de4-1d2c-445d-a4f0-e3fd900e218e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: "2026-07-02T13:53:56Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.388s (2.388s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T13:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:57Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d7361af71b4 namespace: kserve-ci-e2e-test resourceVersion: "14732" uid: 46ba0c1d-4a71-487a-9473-e9f7bf8a54a6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: "2026-07-02T13:53:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:57Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d736496bc35 namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: f1f39f6d-c507-4d4e-990c-5b59b8478bb0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: "2026-07-02T13:53:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:57Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d73651525f2 namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 83a8acb2-4884-46e3-a1ea-a2089b3cebcb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:53:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: "2026-07-02T13:54:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:33Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d73833f223c namespace: kserve-ci-e2e-test resourceVersion: "15054" uid: 276d8181-fb53-4652-8013-b29d5459dee8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: "2026-07-02T13:55:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:15Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d85c4f2df65 namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: b0902b2b-c0e6-430a-acfb-ef2aa936ac4c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: "2026-07-02T13:55:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:15Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d85c4f37046 namespace: kserve-ci-e2e-test resourceVersion: "15551" uid: 2b587934-39f4-4c5e-b569-7772316d03e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5 namespace: kserve-ci-e2e-test resourceVersion: "14473" uid: 87f3a61b-c663-45a0-ae75-64c42cbe1695 kind: Event lastTimestamp: "2026-07-02T13:55:18Z" message: 'Readiness probe failed: Get "https://10.133.0.18:8643/healthz": dial tcp 10.133.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:18Z" name: error-404-isvc-40523-predictor-57f4887bdf-rkfb5.18be7d865f5a853d namespace: kserve-ci-e2e-test resourceVersion: "15610" uid: 87d4dae7-de65-446d-bb05-25239a654825 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-40523-predictor-57f4887bdf namespace: kserve-ci-e2e-test resourceVersion: "14461" uid: 74cf5502-dd19-4e86-a4b7-bb15a727d6f7 kind: Event lastTimestamp: "2026-07-02T13:53:41Z" message: 'Created pod: error-404-isvc-40523-predictor-57f4887bdf-rkfb5' metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:41Z" name: error-404-isvc-40523-predictor-57f4887bdf.18be7d6fd639e85f namespace: kserve-ci-e2e-test resourceVersion: "14471" uid: 5736585c-7d53-4363-a20d-5f6940cb9065 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-40523-predictor namespace: kserve-ci-e2e-test resourceVersion: "14458" uid: 3c234ef9-9fe7-4b1c-8779-efa03837a512 kind: Event lastTimestamp: "2026-07-02T13:53:41Z" message: Scaled up replica set error-404-isvc-40523-predictor-57f4887bdf from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:41Z" name: error-404-isvc-40523-predictor.18be7d6fd4248334 namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: d5c53f9e-fea6-4c8e-ba67-5e283155797b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-40523-predictor namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: 1977a5f6-5e3f-4299-8833-616f0bd4c86c kind: Event lastTimestamp: "2026-07-02T13:53:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:56Z" name: error-404-isvc-40523-predictor.18be7d735345863f namespace: kserve-ci-e2e-test resourceVersion: "14721" uid: fba00b14-676a-49c0-8c0a-365d7a3d662b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-40523-predictor namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: 1977a5f6-5e3f-4299-8833-616f0bd4c86c kind: Event lastTimestamp: "2026-07-02T13:53:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:56Z" name: error-404-isvc-40523-predictor.18be7d7353d5ed7d namespace: kserve-ci-e2e-test resourceVersion: "14722" uid: a08bb2a4-eb19-47a0-8987-f23a4436685d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:54:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-40523-predictor namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 1977a5f6-5e3f-4299-8833-616f0bd4c86c kind: Event lastTimestamp: "2026-07-02T13:55:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:11Z" name: error-404-isvc-40523-predictor.18be7d76d295224a namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: a2fb8b43-ca04-4c35-be56-7065c4bdcb53 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:54:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-40523-predictor namespace: kserve-ci-e2e-test resourceVersion: "14720" uid: 1977a5f6-5e3f-4299-8833-616f0bd4c86c kind: Event lastTimestamp: "2026-07-02T13:55:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:54:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:11Z" name: error-404-isvc-40523-predictor.18be7d76d305afb3 namespace: kserve-ci-e2e-test resourceVersion: "15409" uid: 16aee108-bf67-40c6-b241-b88c5d5ace77 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-40523 namespace: kserve-ci-e2e-test resourceVersion: "14441" uid: aa0074f4-42a0-43f5-88f7-2ad6fad9d168 kind: Event lastTimestamp: "2026-07-02T13:54:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:54:43Z" name: error-404-isvc-40523.18be7d6fd42df6be namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: bbdbf653-97bc-493e-93fe-64e41984d207 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-40523 namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: aa0074f4-42a0-43f5-88f7-2ad6fad9d168 kind: Event lastTimestamp: "2026-07-02T13:54:43Z" message: InferenceService [error-404-isvc-40523] is Ready metadata: creationTimestamp: "2026-07-02T13:54:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:54:43Z" name: error-404-isvc-40523.18be7d7e3f8748be namespace: kserve-ci-e2e-test resourceVersion: "15153" uid: d24b6af6-56e4-4582-8ee0-3e20b8bb0651 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:55:15.911990Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-68692-predictor-58887df78c-cmj9w namespace: kserve-ci-e2e-test resourceVersion: "15524" uid: 733c3c23-c4a1-4afa-ae84-f6417564974d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-68692-predictor-58887df78c-cmj9w to ip-10-0-133-209.ec2.internal metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:55:15Z" name: error-404-isvc-68692-predictor-58887df78c-cmj9w.18be7d85c145d40c namespace: kserve-ci-e2e-test resourceVersion: "15530" uid: d3bb12d5-0377-4240-944d-c7dec1f7a896 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-68692-predictor-58887df78c-cmj9w namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 733c3c23-c4a1-4afa-ae84-f6417564974d kind: Event lastTimestamp: "2026-07-02T13:55:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-68692-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:16Z" name: error-404-isvc-68692-predictor-58887df78c-cmj9w.18be7d85d143b0a1 namespace: kserve-ci-e2e-test resourceVersion: "15562" uid: 09a189e3-2d58-4601-8669-a817b37c1a2b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-68692-predictor-58887df78c-cmj9w namespace: kserve-ci-e2e-test resourceVersion: "15535" uid: 733c3c23-c4a1-4afa-ae84-f6417564974d kind: Event lastTimestamp: "2026-07-02T13:55:16Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:55:16Z" name: error-404-isvc-68692-predictor-58887df78c-cmj9w.18be7d85ff37dd84 namespace: kserve-ci-e2e-test resourceVersion: "15579" uid: 2515ae28-2379-4bf9-b98e-9a5fa117e8c8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68692-predictor-58887df78c-cmj9w namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 733c3c23-c4a1-4afa-ae84-f6417564974d kind: Event lastTimestamp: "2026-07-02T13:55:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T13:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:16Z" name: error-404-isvc-68692-predictor-58887df78c-cmj9w.18be7d8600d1b7c0 namespace: kserve-ci-e2e-test resourceVersion: "15581" uid: 01d429e2-9a87-4e28-aecf-e00d499598a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68692-predictor-58887df78c-cmj9w namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 733c3c23-c4a1-4afa-ae84-f6417564974d kind: Event lastTimestamp: "2026-07-02T13:55:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:17Z" name: error-404-isvc-68692-predictor-58887df78c-cmj9w.18be7d8604367664 namespace: kserve-ci-e2e-test resourceVersion: "15584" uid: 3530afbb-b8f8-40e9-9706-1a6b2c3befb2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68692-predictor-58887df78c-cmj9w namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 733c3c23-c4a1-4afa-ae84-f6417564974d kind: Event lastTimestamp: "2026-07-02T13:55:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:17Z" name: error-404-isvc-68692-predictor-58887df78c-cmj9w.18be7d8604b12eb3 namespace: kserve-ci-e2e-test resourceVersion: "15585" uid: 55bf1b4f-a8ad-4acd-be95-802069a9c3b9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68692-predictor-58887df78c-cmj9w namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 733c3c23-c4a1-4afa-ae84-f6417564974d kind: Event lastTimestamp: "2026-07-02T13:55:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:17Z" name: error-404-isvc-68692-predictor-58887df78c-cmj9w.18be7d8604bb0eba namespace: kserve-ci-e2e-test resourceVersion: "15587" uid: 2c7152eb-fc11-4149-ae76-903262e4f860 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68692-predictor-58887df78c-cmj9w namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 733c3c23-c4a1-4afa-ae84-f6417564974d kind: Event lastTimestamp: "2026-07-02T13:55:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:17Z" name: error-404-isvc-68692-predictor-58887df78c-cmj9w.18be7d8607ac463e namespace: kserve-ci-e2e-test resourceVersion: "15588" uid: 8d1be6fa-2f20-418f-95c6-63035dee68a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68692-predictor-58887df78c-cmj9w namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 733c3c23-c4a1-4afa-ae84-f6417564974d kind: Event lastTimestamp: "2026-07-02T13:55:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:17Z" name: error-404-isvc-68692-predictor-58887df78c-cmj9w.18be7d8608278cdf namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 4fe0e85c-ad1b-4bd5-abd8-38ed5855a12f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68692-predictor-58887df78c-cmj9w namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 733c3c23-c4a1-4afa-ae84-f6417564974d kind: Event lastTimestamp: "2026-07-02T14:04:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:33Z" name: error-404-isvc-68692-predictor-58887df78c-cmj9w.18be7d8635044e04 namespace: kserve-ci-e2e-test resourceVersion: "20015" uid: 1b060846-24a5-4417-8a55-d99a3a7e680a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68692-predictor-58887df78c-cmj9w namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 733c3c23-c4a1-4afa-ae84-f6417564974d kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:30Z" name: error-404-isvc-68692-predictor-58887df78c-cmj9w.18be7e06edcedd95 namespace: kserve-ci-e2e-test resourceVersion: "19949" uid: b07564e2-1dce-47d9-aaf9-77849117b6f5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68692-predictor-58887df78c-cmj9w namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 733c3c23-c4a1-4afa-ae84-f6417564974d kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:30Z" name: error-404-isvc-68692-predictor-58887df78c-cmj9w.18be7e06edcf5af4 namespace: kserve-ci-e2e-test resourceVersion: "19952" uid: 99cf774c-3491-4208-a0ef-4cd043bfb49c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68692-predictor-58887df78c-cmj9w namespace: kserve-ci-e2e-test resourceVersion: "15526" uid: 733c3c23-c4a1-4afa-ae84-f6417564974d kind: Event lastTimestamp: "2026-07-02T14:04:33Z" message: 'Readiness probe failed: Get "https://10.133.0.19:8643/healthz": dial tcp 10.133.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:33Z" name: error-404-isvc-68692-predictor-58887df78c-cmj9w.18be7e07a964d298 namespace: kserve-ci-e2e-test resourceVersion: "20014" uid: 71f7b1b2-72ee-4b59-bf84-675ab112010e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-68692-predictor-58887df78c namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 11f8f64a-d967-4ddf-a69c-4e9f5430db34 kind: Event lastTimestamp: "2026-07-02T13:55:15Z" message: 'Created pod: error-404-isvc-68692-predictor-58887df78c-cmj9w' metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:15Z" name: error-404-isvc-68692-predictor-58887df78c.18be7d85c0c28c61 namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: 34351ab6-4104-426c-b50c-c7521837fb48 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-68692-predictor namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: 1de38312-5462-4b92-943c-d367355c9943 kind: Event lastTimestamp: "2026-07-02T13:55:15Z" message: Scaled up replica set error-404-isvc-68692-predictor-58887df78c from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:15Z" name: error-404-isvc-68692-predictor.18be7d85bec619ab namespace: kserve-ci-e2e-test resourceVersion: "15517" uid: 148231a5-120b-4a80-85c9-ea8073f23a78 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68692-predictor namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: 909d5e4f-9977-43b1-9c2a-3f34890954a2 kind: Event lastTimestamp: "2026-07-02T13:55:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:30Z" name: error-404-isvc-68692-predictor.18be7d893d835ad4 namespace: kserve-ci-e2e-test resourceVersion: "15737" uid: 46c220e1-24b8-4e51-b162-f9e8d516a4f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68692-predictor namespace: kserve-ci-e2e-test resourceVersion: "15513" uid: 909d5e4f-9977-43b1-9c2a-3f34890954a2 kind: Event lastTimestamp: "2026-07-02T13:55:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:30Z" name: error-404-isvc-68692-predictor.18be7d893dfafd40 namespace: kserve-ci-e2e-test resourceVersion: "15739" uid: 38f8579d-c8c1-44ed-8b62-b891465e7424 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:55:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68692-predictor namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: 909d5e4f-9977-43b1-9c2a-3f34890954a2 kind: Event lastTimestamp: "2026-07-02T13:56:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:30Z" name: error-404-isvc-68692-predictor.18be7d8cbcc19a59 namespace: kserve-ci-e2e-test resourceVersion: "16497" uid: c3a40119-58c7-4b97-858e-5c385022e1c3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:55:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68692-predictor namespace: kserve-ci-e2e-test resourceVersion: "15738" uid: 909d5e4f-9977-43b1-9c2a-3f34890954a2 kind: Event lastTimestamp: "2026-07-02T13:56:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:30Z" name: error-404-isvc-68692-predictor.18be7d8cbd33a892 namespace: kserve-ci-e2e-test resourceVersion: "16498" uid: 39b6ef47-858d-4aa0-b1e7-f9c02b53450e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68692 namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: e5ded4c7-5d02-42b9-a3d3-2565be97d079 kind: Event lastTimestamp: "2026-07-02T13:56:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:56:03Z" name: error-404-isvc-68692.18be7d85bf0bf0ec namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 29ea1d73-0382-49fa-9a58-f1a333ee1d4a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68692 namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: e5ded4c7-5d02-42b9-a3d3-2565be97d079 kind: Event lastTimestamp: "2026-07-02T13:55:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-68692": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-68692": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:15Z" name: error-404-isvc-68692.18be7d85bfb17622 namespace: kserve-ci-e2e-test resourceVersion: "15521" uid: bbbb4c38-adc6-4dd3-9290-e48210f25f40 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68692 namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: e5ded4c7-5d02-42b9-a3d3-2565be97d079 kind: Event lastTimestamp: "2026-07-02T13:55:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-68692": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:15Z" name: error-404-isvc-68692.18be7d85bfb1cb14 namespace: kserve-ci-e2e-test resourceVersion: "15523" uid: 96ef344f-0471-4914-a42a-5f503077734a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68692 namespace: kserve-ci-e2e-test resourceVersion: "16059" uid: e5ded4c7-5d02-42b9-a3d3-2565be97d079 kind: Event lastTimestamp: "2026-07-02T13:56:03Z" message: InferenceService [error-404-isvc-68692] is Ready metadata: creationTimestamp: "2026-07-02T13:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:56:03Z" name: error-404-isvc-68692.18be7d90f02c1ce3 namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 360745a0-7b6b-4c82-b8d8-1afeeb5dfc44 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:06:37.111040Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: aff672a5-50a2-46cb-85fd-41c7230f155f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6d218-predictor-fc586b6c5-77mmt to ip-10-0-133-209.ec2.internal metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:06:37Z" name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt.18be7e245be68466 namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: 6c0a63b6-d6e1-430c-a2f7-bf142d39f6b4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: aff672a5-50a2-46cb-85fd-41c7230f155f kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6d218-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:37Z" name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt.18be7e2467b102e4 namespace: kserve-ci-e2e-test resourceVersion: "21509" uid: 501a0559-5032-4a1b-8572-0f5fc00bd11b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: aff672a5-50a2-46cb-85fd-41c7230f155f kind: Event lastTimestamp: "2026-07-02T14:06:38Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:06:38Z" name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt.18be7e24994b58ee namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: 4f880a21-66d1-4596-82a1-d5f6c2676303 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: aff672a5-50a2-46cb-85fd-41c7230f155f kind: Event lastTimestamp: "2026-07-02T14:06:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:38Z" name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt.18be7e249bebb676 namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: 42ee80d9-abb6-4648-83f0-9fab31cccbd2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: aff672a5-50a2-46cb-85fd-41c7230f155f kind: Event lastTimestamp: "2026-07-02T14:06:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:38Z" name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt.18be7e249f685c34 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: ee121d68-7c27-47d6-8f4d-58f1a159e3e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: aff672a5-50a2-46cb-85fd-41c7230f155f kind: Event lastTimestamp: "2026-07-02T14:06:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:38Z" name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt.18be7e249fe88802 namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: e7ff3225-80c2-4a4a-9775-b362139eeadd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: aff672a5-50a2-46cb-85fd-41c7230f155f kind: Event lastTimestamp: "2026-07-02T14:06:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:38Z" name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt.18be7e249ff2a5d8 namespace: kserve-ci-e2e-test resourceVersion: "21535" uid: d13e3437-0d27-444f-9e7f-d9069e45606d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: aff672a5-50a2-46cb-85fd-41c7230f155f kind: Event lastTimestamp: "2026-07-02T14:06:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:38Z" name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt.18be7e24a2ace7ad namespace: kserve-ci-e2e-test resourceVersion: "21536" uid: b6f4b868-8779-415a-9270-b5cd21e0bf89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: aff672a5-50a2-46cb-85fd-41c7230f155f kind: Event lastTimestamp: "2026-07-02T14:06:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:38Z" name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt.18be7e24a322006b namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: 9a7db98e-586f-4fad-a2de-c19b11d954a5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: aff672a5-50a2-46cb-85fd-41c7230f155f kind: Event lastTimestamp: "2026-07-02T14:07:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:15Z" name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt.18be7e24e31a3263 namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: a5af4c25-b8b4-4f79-99f7-cc177d3c06e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: aff672a5-50a2-46cb-85fd-41c7230f155f kind: Event lastTimestamp: "2026-07-02T14:15:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:52Z" name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt.18be7ea59124d8ae namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: e9f27c0f-e737-4bab-ba85-50f617163356 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt namespace: kserve-ci-e2e-test resourceVersion: "21491" uid: aff672a5-50a2-46cb-85fd-41c7230f155f kind: Event lastTimestamp: "2026-07-02T14:15:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:52Z" name: error-404-isvc-6d218-predictor-fc586b6c5-77mmt.18be7ea591255782 namespace: kserve-ci-e2e-test resourceVersion: "26070" uid: 93dd5544-4622-42a9-94e2-7924fedc2733 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6d218-predictor-fc586b6c5 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 473c08c6-878e-464d-83b9-bb61da760812 kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: 'Created pod: error-404-isvc-6d218-predictor-fc586b6c5-77mmt' metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:37Z" name: error-404-isvc-6d218-predictor-fc586b6c5.18be7e245b79f0b7 namespace: kserve-ci-e2e-test resourceVersion: "21490" uid: b293a887-f1f5-4d9c-8d4b-5d48ab3d7a41 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6d218-predictor namespace: kserve-ci-e2e-test resourceVersion: "21478" uid: b57deb6c-423f-4166-90dc-66d23deaafd7 kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: Scaled up replica set error-404-isvc-6d218-predictor-fc586b6c5 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:37Z" name: error-404-isvc-6d218-predictor.18be7e2459aa9bab namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: fecfb448-6c44-49fc-b3e4-06c85c5abd84 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:06:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6d218-predictor namespace: kserve-ci-e2e-test resourceVersion: "21480" uid: 8deefcbb-71dc-43ca-bd97-89475fbbd4a9 kind: Event lastTimestamp: "2026-07-02T14:07:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:07Z" name: error-404-isvc-6d218-predictor.18be7e27d83ca3d1 namespace: kserve-ci-e2e-test resourceVersion: "21822" uid: bda1b95d-0b2b-4cb1-a0dc-20d48b9a29f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:06:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6d218-predictor namespace: kserve-ci-e2e-test resourceVersion: "21480" uid: 8deefcbb-71dc-43ca-bd97-89475fbbd4a9 kind: Event lastTimestamp: "2026-07-02T14:07:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:06:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:07Z" name: error-404-isvc-6d218-predictor.18be7e27d8bc86ca namespace: kserve-ci-e2e-test resourceVersion: "21823" uid: f53b4f68-de3c-4be2-889a-26c915b2f40c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:07:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6d218-predictor namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: 8deefcbb-71dc-43ca-bd97-89475fbbd4a9 kind: Event lastTimestamp: "2026-07-02T14:07:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:52Z" name: error-404-isvc-6d218-predictor.18be7e2ed64670c9 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 141cbf2d-bb85-477b-8599-faf80f43a6b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:07:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6d218-predictor namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: 8deefcbb-71dc-43ca-bd97-89475fbbd4a9 kind: Event lastTimestamp: "2026-07-02T14:07:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:52Z" name: error-404-isvc-6d218-predictor.18be7e2ed6c803ff namespace: kserve-ci-e2e-test resourceVersion: "22209" uid: 0e3f765d-8c46-4425-b260-8921a319179e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6d218 namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 330444a2-24cc-4821-992d-5aa830f68cc6 kind: Event lastTimestamp: "2026-07-02T14:07:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:07:25Z" name: error-404-isvc-6d218.18be7e2459c6b105 namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: 0c39fe27-2315-456d-9b0b-ebd81787ee21 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6d218 namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 330444a2-24cc-4821-992d-5aa830f68cc6 kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: 'Failed to update status for InferenceService "error-404-isvc-6d218": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6d218": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:37Z" name: error-404-isvc-6d218.18be7e245a599378 namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: d424f75c-e2d9-49cb-8318-6dedcf563737 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6d218 namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 330444a2-24cc-4821-992d-5aa830f68cc6 kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-6d218": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:37Z" name: error-404-isvc-6d218.18be7e245a59dff9 namespace: kserve-ci-e2e-test resourceVersion: "21488" uid: d5c7f781-260d-4ba6-a427-e80075f849a0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6d218 namespace: kserve-ci-e2e-test resourceVersion: "21979" uid: 330444a2-24cc-4821-992d-5aa830f68cc6 kind: Event lastTimestamp: "2026-07-02T14:07:25Z" message: InferenceService [error-404-isvc-6d218] is Ready metadata: creationTimestamp: "2026-07-02T14:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:07:25Z" name: error-404-isvc-6d218.18be7e2fa04ace66 namespace: kserve-ci-e2e-test resourceVersion: "21980" uid: 41b35c05-ab21-4353-9a5b-d9173e5ffabf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:16:06.263024Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78590-predictor-64948cf447-bmtpr namespace: kserve-ci-e2e-test resourceVersion: "26334" uid: 99bd2872-b70a-4d62-bddf-1947e802d6ed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-78590-predictor-64948cf447-bmtpr to ip-10-0-133-209.ec2.internal metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-78590-predictor-64948cf447-bmtpr.18be7ea8e0008185 namespace: kserve-ci-e2e-test resourceVersion: "26340" uid: 5039ea7b-9106-4e97-b162-7669a129bdac reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-78590-predictor-64948cf447-bmtpr namespace: kserve-ci-e2e-test resourceVersion: "26352" uid: 99bd2872-b70a-4d62-bddf-1947e802d6ed kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-78590-predictor-64948cf447-bmtpr.18be7ea8f9b57e5f namespace: kserve-ci-e2e-test resourceVersion: "26378" uid: b3dab9da-b5c5-438b-934e-5ab0da40c03a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78590-predictor-64948cf447-bmtpr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 99bd2872-b70a-4d62-bddf-1947e802d6ed kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-78590-predictor-64948cf447-bmtpr.18be7ea8fb3d7987 namespace: kserve-ci-e2e-test resourceVersion: "26382" uid: ae51b0ca-c0d3-432a-b669-69c6b08009d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78590-predictor-64948cf447-bmtpr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 99bd2872-b70a-4d62-bddf-1947e802d6ed kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-78590-predictor-64948cf447-bmtpr.18be7ea8ff07da00 namespace: kserve-ci-e2e-test resourceVersion: "26388" uid: 46864124-0088-4c8a-bc78-bc8db772c775 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78590-predictor-64948cf447-bmtpr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 99bd2872-b70a-4d62-bddf-1947e802d6ed kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-78590-predictor-64948cf447-bmtpr.18be7ea8ff8508b7 namespace: kserve-ci-e2e-test resourceVersion: "26389" uid: 934a08b0-ee31-4864-a4a8-10b64204e306 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78590-predictor-64948cf447-bmtpr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 99bd2872-b70a-4d62-bddf-1947e802d6ed kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-78590-predictor-64948cf447-bmtpr.18be7ea8ff90269a namespace: kserve-ci-e2e-test resourceVersion: "26390" uid: 34c95cac-2d06-4789-8944-7ef033091b0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78590-predictor-64948cf447-bmtpr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 99bd2872-b70a-4d62-bddf-1947e802d6ed kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-78590-predictor-64948cf447-bmtpr.18be7ea90300dacb namespace: kserve-ci-e2e-test resourceVersion: "26395" uid: f0093830-d9d4-48fe-a524-f28e9dceed13 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78590-predictor-64948cf447-bmtpr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 99bd2872-b70a-4d62-bddf-1947e802d6ed kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-78590-predictor-64948cf447-bmtpr.18be7ea9037d3704 namespace: kserve-ci-e2e-test resourceVersion: "26396" uid: 47ec8f5c-8b1f-4f39-a385-7d831dc2f5a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:16:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78590-predictor-64948cf447-bmtpr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 99bd2872-b70a-4d62-bddf-1947e802d6ed kind: Event lastTimestamp: "2026-07-02T14:25:23Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:16:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:23Z" name: error-404-isvc-78590-predictor-64948cf447-bmtpr.18be7ea92b3b43d0 namespace: kserve-ci-e2e-test resourceVersion: "30483" uid: 438f9f18-328f-43eb-a205-1594c39eb8a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-78590-predictor-64948cf447-bmtpr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 99bd2872-b70a-4d62-bddf-1947e802d6ed kind: Event lastTimestamp: "2026-07-02T14:25:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:21Z" name: error-404-isvc-78590-predictor-64948cf447-bmtpr.18be7f2a09057684 namespace: kserve-ci-e2e-test resourceVersion: "30459" uid: 7c5cea29-a1d8-4a6f-8f24-4c7440787f73 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78590-predictor-64948cf447-bmtpr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 99bd2872-b70a-4d62-bddf-1947e802d6ed kind: Event lastTimestamp: "2026-07-02T14:25:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:25:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:21Z" name: error-404-isvc-78590-predictor-64948cf447-bmtpr.18be7f2a090619ab namespace: kserve-ci-e2e-test resourceVersion: "30460" uid: e5f02bb6-5d7c-4640-8967-a2d19d0a1f0b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-78590-predictor-64948cf447-bmtpr namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 99bd2872-b70a-4d62-bddf-1947e802d6ed kind: Event lastTimestamp: "2026-07-02T14:25:23Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:25:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:23Z" name: error-404-isvc-78590-predictor-64948cf447-bmtpr.18be7f2a9fa7eae9 namespace: kserve-ci-e2e-test resourceVersion: "30482" uid: cbf0cfa8-dc59-4ff5-aab3-7b7eee1f5f87 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-78590-predictor-64948cf447 namespace: kserve-ci-e2e-test resourceVersion: "26328" uid: 76bd9f8f-de0e-4620-863f-0902ad926370 kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: 'Created pod: error-404-isvc-78590-predictor-64948cf447-bmtpr' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-78590-predictor-64948cf447.18be7ea8df6df174 namespace: kserve-ci-e2e-test resourceVersion: "26336" uid: 62fafa6a-ad6b-40b7-821d-29d3e1c6b349 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-78590-predictor namespace: kserve-ci-e2e-test resourceVersion: "26326" uid: 9592118e-3c6e-49d3-a14e-77a1585c52ad kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Scaled up replica set error-404-isvc-78590-predictor-64948cf447 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-78590-predictor.18be7ea8dd82025a namespace: kserve-ci-e2e-test resourceVersion: "26329" uid: 257edeeb-804d-41dd-b25d-dccda5800ae7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:16:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78590-predictor namespace: kserve-ci-e2e-test resourceVersion: "26327" uid: e28d6081-d642-40c2-8b09-80a6b18262f7 kind: Event lastTimestamp: "2026-07-02T14:17:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:16:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:06Z" name: error-404-isvc-78590-predictor.18be7eac5bf7aaae namespace: kserve-ci-e2e-test resourceVersion: "26988" uid: b3a4234d-b7b1-4935-b41c-c052c36fdcf3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:16:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78590-predictor namespace: kserve-ci-e2e-test resourceVersion: "26327" uid: e28d6081-d642-40c2-8b09-80a6b18262f7 kind: Event lastTimestamp: "2026-07-02T14:17:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:16:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:06Z" name: error-404-isvc-78590-predictor.18be7eac5c6763d0 namespace: kserve-ci-e2e-test resourceVersion: "26989" uid: 151387d5-cc43-41e5-bfcd-bfeed41a3fa5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78590-predictor namespace: kserve-ci-e2e-test resourceVersion: "26535" uid: e28d6081-d642-40c2-8b09-80a6b18262f7 kind: Event lastTimestamp: "2026-07-02T14:16:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:16:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:16:36Z" name: error-404-isvc-78590-predictor.18be7eafdb9a2c79 namespace: kserve-ci-e2e-test resourceVersion: "26660" uid: 941cf270-d712-4399-929c-4d5f4ecab04c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-78590-predictor namespace: kserve-ci-e2e-test resourceVersion: "26535" uid: e28d6081-d642-40c2-8b09-80a6b18262f7 kind: Event lastTimestamp: "2026-07-02T14:16:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:16:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:16:36Z" name: error-404-isvc-78590-predictor.18be7eafdc1e2613 namespace: kserve-ci-e2e-test resourceVersion: "26661" uid: 751b7004-129b-4f36-a996-1cf89f90574d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78590 namespace: kserve-ci-e2e-test resourceVersion: "26315" uid: d7695d1f-9812-4f81-93f4-56857507e7bd kind: Event lastTimestamp: "2026-07-02T14:16:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:16:53Z" name: error-404-isvc-78590.18be7ea8dd84a8b8 namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: e5e19032-ba85-4434-a6e3-aaee84901b3d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78590 namespace: kserve-ci-e2e-test resourceVersion: "26315" uid: d7695d1f-9812-4f81-93f4-56857507e7bd kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: 'Failed to update status for InferenceService "error-404-isvc-78590": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-78590": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-78590.18be7ea8ddfe6461 namespace: kserve-ci-e2e-test resourceVersion: "26331" uid: fcac0b09-b9a3-4d82-a7b6-c1b8f2e309a3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78590 namespace: kserve-ci-e2e-test resourceVersion: "26315" uid: d7695d1f-9812-4f81-93f4-56857507e7bd kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-78590": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:16:06Z" name: error-404-isvc-78590.18be7ea8ddfe9dab namespace: kserve-ci-e2e-test resourceVersion: "26333" uid: 86bf3fcb-a58a-4ee5-9e57-4581395c951c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-78590 namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: d7695d1f-9812-4f81-93f4-56857507e7bd kind: Event lastTimestamp: "2026-07-02T14:16:53Z" message: InferenceService [error-404-isvc-78590] is Ready metadata: creationTimestamp: "2026-07-02T14:16:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:16:53Z" name: error-404-isvc-78590.18be7eb3e7319146 namespace: kserve-ci-e2e-test resourceVersion: "26863" uid: 03099ef4-c0d3-4cd0-9962-bab6bcd611c8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:05:40.944416Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: bf5c8734-5d7c-47a7-a030-d10ef03f926a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7b7ab-predictor-7d65578975-ghbgv to ip-10-0-133-209.ec2.internal metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:05:40Z" name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv.18be7e17481bcfd4 namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: ffa2f806-f70d-4beb-a18d-183bb9d95e81 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: bf5c8734-5d7c-47a7-a030-d10ef03f926a kind: Event lastTimestamp: "2026-07-02T14:05:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7b7ab-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:41Z" name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv.18be7e1754d02cb2 namespace: kserve-ci-e2e-test resourceVersion: "20838" uid: 5733433c-68ab-4c79-b660-4f8a6ca386ad reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: bf5c8734-5d7c-47a7-a030-d10ef03f926a kind: Event lastTimestamp: "2026-07-02T14:05:41Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:05:41Z" name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv.18be7e1785dc51d9 namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 75c6e7ea-b1f0-49dd-8a04-04fb0df86271 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: bf5c8734-5d7c-47a7-a030-d10ef03f926a kind: Event lastTimestamp: "2026-07-02T14:05:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:42Z" name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv.18be7e17875ebc3b namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 9e8a3e8a-86f6-4dee-aff8-d91ab3da28d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: bf5c8734-5d7c-47a7-a030-d10ef03f926a kind: Event lastTimestamp: "2026-07-02T14:05:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:42Z" name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv.18be7e178a4b9659 namespace: kserve-ci-e2e-test resourceVersion: "20867" uid: 307ecc41-2add-4563-9590-b6ec9bbac9d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: bf5c8734-5d7c-47a7-a030-d10ef03f926a kind: Event lastTimestamp: "2026-07-02T14:05:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:42Z" name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv.18be7e178ac769a9 namespace: kserve-ci-e2e-test resourceVersion: "20868" uid: 55158216-8452-4b12-bbb1-fb58ae14ca5c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: bf5c8734-5d7c-47a7-a030-d10ef03f926a kind: Event lastTimestamp: "2026-07-02T14:05:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:42Z" name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv.18be7e178ad21b7f namespace: kserve-ci-e2e-test resourceVersion: "20869" uid: 61315be8-52e6-490e-99c1-9de800e75cba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: bf5c8734-5d7c-47a7-a030-d10ef03f926a kind: Event lastTimestamp: "2026-07-02T14:05:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:42Z" name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv.18be7e178e3f2f13 namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: ed26c17d-1b99-49ea-ba57-52da9a546e70 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: bf5c8734-5d7c-47a7-a030-d10ef03f926a kind: Event lastTimestamp: "2026-07-02T14:05:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:42Z" name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv.18be7e178ebb5132 namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: f5ee9985-49e4-4391-8cbd-0d557dbe027b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: bf5c8734-5d7c-47a7-a030-d10ef03f926a kind: Event lastTimestamp: "2026-07-02T14:14:59Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:59Z" name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv.18be7e17cc2d18e2 namespace: kserve-ci-e2e-test resourceVersion: "25251" uid: b0a10fb5-fc40-4a59-819c-c842b741c428 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: bf5c8734-5d7c-47a7-a030-d10ef03f926a kind: Event lastTimestamp: "2026-07-02T14:14:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:56Z" name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv.18be7e98848f8d47 namespace: kserve-ci-e2e-test resourceVersion: "25190" uid: 15702e6d-9330-4235-bc1e-33f05fb6cd0c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: bf5c8734-5d7c-47a7-a030-d10ef03f926a kind: Event lastTimestamp: "2026-07-02T14:14:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:56Z" name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv.18be7e98848ff6a2 namespace: kserve-ci-e2e-test resourceVersion: "25193" uid: 6ddf1447-e7cc-4ffe-9861-28ca4058cf17 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: bf5c8734-5d7c-47a7-a030-d10ef03f926a kind: Event lastTimestamp: "2026-07-02T14:14:59Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:14:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:59Z" name: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv.18be7e9940a1d6ea namespace: kserve-ci-e2e-test resourceVersion: "25250" uid: ea0f5599-2876-484e-bb64-3660a588492b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7b7ab-predictor-7d65578975 namespace: kserve-ci-e2e-test resourceVersion: "20811" uid: 8d50754a-cbcb-43d5-876f-af36afb18aa0 kind: Event lastTimestamp: "2026-07-02T14:05:40Z" message: 'Created pod: error-404-isvc-7b7ab-predictor-7d65578975-ghbgv' metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:40Z" name: error-404-isvc-7b7ab-predictor-7d65578975.18be7e1747a4a7ce namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: c3f8dc16-cafe-490e-8a3d-cc5175dc9db7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7b7ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "20808" uid: 84e189a8-b33d-4814-8d70-db784dac462b kind: Event lastTimestamp: "2026-07-02T14:05:40Z" message: Scaled up replica set error-404-isvc-7b7ab-predictor-7d65578975 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:40Z" name: error-404-isvc-7b7ab-predictor.18be7e1746085fa1 namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: ebce7894-78ca-4cc2-bb5e-cf4fdabde187 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b7ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: 41841ac0-f15b-476d-9c28-30c53fde835a kind: Event lastTimestamp: "2026-07-02T14:06:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:10Z" name: error-404-isvc-7b7ab-predictor.18be7e1ac485872f namespace: kserve-ci-e2e-test resourceVersion: "21121" uid: a9eb7291-49ab-4377-b96b-097585e1d8d5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b7ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: 41841ac0-f15b-476d-9c28-30c53fde835a kind: Event lastTimestamp: "2026-07-02T14:06:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:10Z" name: error-404-isvc-7b7ab-predictor.18be7e1ac4edeafc namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 25cf3566-7417-4713-b00a-be5d776d8c7d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:06:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b7ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: 41841ac0-f15b-476d-9c28-30c53fde835a kind: Event lastTimestamp: "2026-07-02T14:06:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:40Z" name: error-404-isvc-7b7ab-predictor.18be7e21c29f954c namespace: kserve-ci-e2e-test resourceVersion: "21595" uid: e48e47f2-ccc2-41e2-bb89-34eb74a56aec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:06:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7b7ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "21004" uid: 41841ac0-f15b-476d-9c28-30c53fde835a kind: Event lastTimestamp: "2026-07-02T14:06:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:40Z" name: error-404-isvc-7b7ab-predictor.18be7e21c3094ebb namespace: kserve-ci-e2e-test resourceVersion: "21597" uid: a8bb0f81-3a4b-4f1a-ba02-2b49c1c0e0d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b7ab namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: c34242b8-2411-4633-8557-50b0580b1a35 kind: Event lastTimestamp: "2026-07-02T14:06:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:29Z" name: error-404-isvc-7b7ab.18be7e17460b8c83 namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: b4ffcdc3-3dd0-4e40-a972-fcd861da428c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b7ab namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: c34242b8-2411-4633-8557-50b0580b1a35 kind: Event lastTimestamp: "2026-07-02T14:05:40Z" message: 'Failed to update status for InferenceService "error-404-isvc-7b7ab": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7b7ab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:40Z" name: error-404-isvc-7b7ab.18be7e17468ed07f namespace: kserve-ci-e2e-test resourceVersion: "20817" uid: 3e0f5abe-8601-4ed5-821c-15da42415202 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b7ab namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: c34242b8-2411-4633-8557-50b0580b1a35 kind: Event lastTimestamp: "2026-07-02T14:05:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7b7ab": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:40Z" name: error-404-isvc-7b7ab.18be7e17468f2ba9 namespace: kserve-ci-e2e-test resourceVersion: "20818" uid: af94264a-b338-4c78-b89a-8947ffaccd60 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7b7ab namespace: kserve-ci-e2e-test resourceVersion: "21337" uid: c34242b8-2411-4633-8557-50b0580b1a35 kind: Event lastTimestamp: "2026-07-02T14:06:29Z" message: InferenceService [error-404-isvc-7b7ab] is Ready metadata: creationTimestamp: "2026-07-02T14:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:29Z" name: error-404-isvc-7b7ab.18be7e22875b0adc namespace: kserve-ci-e2e-test resourceVersion: "21338" uid: 52f3a078-8335-41b5-b2fb-3d56085ff067 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:56:11.978465Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-88827-predictor-66b8576947-28xrd namespace: kserve-ci-e2e-test resourceVersion: "16219" uid: f12742bd-4c26-4ef1-9c8d-e41010794a1c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-88827-predictor-66b8576947-28xrd to ip-10-0-133-209.ec2.internal metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:56:11Z" name: error-404-isvc-88827-predictor-66b8576947-28xrd.18be7d92cf186832 namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: 7ff1708e-b451-4f8f-a025-aa21e1972f7b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-88827-predictor-66b8576947-28xrd namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: f12742bd-4c26-4ef1-9c8d-e41010794a1c kind: Event lastTimestamp: "2026-07-02T13:56:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-88827-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:12Z" name: error-404-isvc-88827-predictor-66b8576947-28xrd.18be7d92dd4ebfca namespace: kserve-ci-e2e-test resourceVersion: "16254" uid: 082204f7-e8d6-4fb0-a22e-3f9f81c423e0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-88827-predictor-66b8576947-28xrd namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: f12742bd-4c26-4ef1-9c8d-e41010794a1c kind: Event lastTimestamp: "2026-07-02T13:56:13Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:56:13Z" name: error-404-isvc-88827-predictor-66b8576947-28xrd.18be7d930c9256d6 namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 41901a4b-1059-4c56-82da-f957bc3a5937 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-88827-predictor-66b8576947-28xrd namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: f12742bd-4c26-4ef1-9c8d-e41010794a1c kind: Event lastTimestamp: "2026-07-02T13:56:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T13:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:13Z" name: error-404-isvc-88827-predictor-66b8576947-28xrd.18be7d930de8b96b namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 8b42518c-b41d-4fd5-89e5-eb51c37306a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-88827-predictor-66b8576947-28xrd namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: f12742bd-4c26-4ef1-9c8d-e41010794a1c kind: Event lastTimestamp: "2026-07-02T13:56:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:13Z" name: error-404-isvc-88827-predictor-66b8576947-28xrd.18be7d93113eab3c namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 403a0642-208d-4aa3-ba97-bf2746c0c385 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-88827-predictor-66b8576947-28xrd namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: f12742bd-4c26-4ef1-9c8d-e41010794a1c kind: Event lastTimestamp: "2026-07-02T13:56:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:13Z" name: error-404-isvc-88827-predictor-66b8576947-28xrd.18be7d9311b72ab4 namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: c27333e9-0b85-4070-862b-6618823f252d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-88827-predictor-66b8576947-28xrd namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: f12742bd-4c26-4ef1-9c8d-e41010794a1c kind: Event lastTimestamp: "2026-07-02T13:56:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:13Z" name: error-404-isvc-88827-predictor-66b8576947-28xrd.18be7d9311c1f6e1 namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: f4919fde-6773-4101-ab02-bda40801e2b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-88827-predictor-66b8576947-28xrd namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: f12742bd-4c26-4ef1-9c8d-e41010794a1c kind: Event lastTimestamp: "2026-07-02T13:56:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:13Z" name: error-404-isvc-88827-predictor-66b8576947-28xrd.18be7d9314df2432 namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: 12c8f4b2-4d78-4805-9338-976251f99b42 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-88827-predictor-66b8576947-28xrd namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: f12742bd-4c26-4ef1-9c8d-e41010794a1c kind: Event lastTimestamp: "2026-07-02T13:56:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:13Z" name: error-404-isvc-88827-predictor-66b8576947-28xrd.18be7d93155927d0 namespace: kserve-ci-e2e-test resourceVersion: "16283" uid: 866a8041-f227-4890-9b8c-7a17459106b9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-88827-predictor-66b8576947-28xrd namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: f12742bd-4c26-4ef1-9c8d-e41010794a1c kind: Event lastTimestamp: "2026-07-02T13:56:50Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:50Z" name: error-404-isvc-88827-predictor-66b8576947-28xrd.18be7d934bb4c344 namespace: kserve-ci-e2e-test resourceVersion: "16643" uid: fb9c34da-897b-48ef-9b2b-0aad6ccbc51b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-88827-predictor-66b8576947-28xrd namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: f12742bd-4c26-4ef1-9c8d-e41010794a1c kind: Event lastTimestamp: "2026-07-02T14:05:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:26Z" name: error-404-isvc-88827-predictor-66b8576947-28xrd.18be7e13f73febc3 namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: c4ff4078-c155-47fe-97db-de8d9ffaccd1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-88827-predictor-66b8576947-28xrd namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: f12742bd-4c26-4ef1-9c8d-e41010794a1c kind: Event lastTimestamp: "2026-07-02T14:05:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:26Z" name: error-404-isvc-88827-predictor-66b8576947-28xrd.18be7e13f740a0d5 namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: 726650f8-9f30-4894-a30a-a3e05fdc8332 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-88827-predictor-66b8576947 namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: 887c6134-6aaf-48d2-9fcd-21a1b4cf8665 kind: Event lastTimestamp: "2026-07-02T13:56:11Z" message: 'Created pod: error-404-isvc-88827-predictor-66b8576947-28xrd' metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:11Z" name: error-404-isvc-88827-predictor-66b8576947.18be7d92ceafb36e namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: 9c689a96-2052-44cb-a01f-6171c5af2a10 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-88827-predictor namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 1cfa46ca-9fe9-495b-93a1-69f533ccbe84 kind: Event lastTimestamp: "2026-07-02T13:56:11Z" message: Scaled up replica set error-404-isvc-88827-predictor-66b8576947 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:11Z" name: error-404-isvc-88827-predictor.18be7d92cd0c26b4 namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: 46aaeee4-e7f3-4155-88fc-a2fa9d129263 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:56:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-88827-predictor namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: a03c3a4b-19d8-451d-91b8-f6d738e52fe7 kind: Event lastTimestamp: "2026-07-02T13:56:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:56:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:41Z" name: error-404-isvc-88827-predictor.18be7d964b834aa5 namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: 8b3cb54c-6cfd-4489-afb1-14696330665c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:56:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-88827-predictor namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: a03c3a4b-19d8-451d-91b8-f6d738e52fe7 kind: Event lastTimestamp: "2026-07-02T13:56:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:56:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:41Z" name: error-404-isvc-88827-predictor.18be7d964bf15b51 namespace: kserve-ci-e2e-test resourceVersion: "16584" uid: fc6fb05b-2720-4098-98ec-9aa152447aae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:56:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-88827-predictor namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: a03c3a4b-19d8-451d-91b8-f6d738e52fe7 kind: Event lastTimestamp: "2026-07-02T13:57:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:27Z" name: error-404-isvc-88827-predictor.18be7d9d4992b09d namespace: kserve-ci-e2e-test resourceVersion: "16968" uid: 66a5a3fd-0887-4e5c-a4b2-131a23c095ed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:56:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-88827-predictor namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: a03c3a4b-19d8-451d-91b8-f6d738e52fe7 kind: Event lastTimestamp: "2026-07-02T13:57:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:27Z" name: error-404-isvc-88827-predictor.18be7d9d4a0e2d26 namespace: kserve-ci-e2e-test resourceVersion: "16969" uid: d41366ae-fe75-4b02-8e02-6ef0d1e002b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:56:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-88827 namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 8727fd64-9829-428a-a69a-a9e1e3b00854 kind: Event lastTimestamp: "2026-07-02T13:57:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:57:00Z" name: error-404-isvc-88827.18be7d92cd125629 namespace: kserve-ci-e2e-test resourceVersion: "16733" uid: 892615dc-dcd9-4ec8-a5bf-41e8e0e296ee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-88827 namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 8727fd64-9829-428a-a69a-a9e1e3b00854 kind: Event lastTimestamp: "2026-07-02T13:56:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-88827": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-88827": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:56:11Z" name: error-404-isvc-88827.18be7d92cd902202 namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 5d52707a-c419-4c64-adee-21ddb6392602 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-88827 namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 8727fd64-9829-428a-a69a-a9e1e3b00854 kind: Event lastTimestamp: "2026-07-02T13:56:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-88827": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:56:11Z" name: error-404-isvc-88827.18be7d92cd905268 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 2c5dfae1-ffe4-4664-afc6-3ac479acc2cc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-88827 namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 8727fd64-9829-428a-a69a-a9e1e3b00854 kind: Event lastTimestamp: "2026-07-02T13:57:00Z" message: InferenceService [error-404-isvc-88827] is Ready metadata: creationTimestamp: "2026-07-02T13:57:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:57:00Z" name: error-404-isvc-88827.18be7d9e06f47eb2 namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: ac1a7a00-4cd0-49a1-bb16-7a80518c1273 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:15:51.957642Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: 373c3199-3158-4e4c-8eab-6105fc0d4734 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm to ip-10-0-133-209.ec2.internal metadata: creationTimestamp: "2026-07-02T14:15:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:15:51Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm.18be7ea58b55cab6 namespace: kserve-ci-e2e-test resourceVersion: "26047" uid: 37e1772d-ed91-4e4e-b753-6bf65ba17ce4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 373c3199-3158-4e4c-8eab-6105fc0d4734 kind: Event lastTimestamp: "2026-07-02T14:15:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8d2ee-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:52Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm.18be7ea594239521 namespace: kserve-ci-e2e-test resourceVersion: "26073" uid: afe1a95f-f1eb-4464-a0df-7508c3099d40 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm namespace: kserve-ci-e2e-test resourceVersion: "26052" uid: 373c3199-3158-4e4c-8eab-6105fc0d4734 kind: Event lastTimestamp: "2026-07-02T14:15:52Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:15:52Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm.18be7ea5c89e5315 namespace: kserve-ci-e2e-test resourceVersion: "26097" uid: 62a23ab8-5da1-4c52-904f-983e259c0db5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 373c3199-3158-4e4c-8eab-6105fc0d4734 kind: Event lastTimestamp: "2026-07-02T14:15:53Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:15:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:53Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm.18be7ea5ca0dfa7a namespace: kserve-ci-e2e-test resourceVersion: "26099" uid: fee5f666-ecc8-4883-ace7-a0211bb142c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 373c3199-3158-4e4c-8eab-6105fc0d4734 kind: Event lastTimestamp: "2026-07-02T14:15:53Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:15:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:53Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm.18be7ea5cd055cae namespace: kserve-ci-e2e-test resourceVersion: "26100" uid: fc77085b-fa26-4aae-bf35-4862d99ec832 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 373c3199-3158-4e4c-8eab-6105fc0d4734 kind: Event lastTimestamp: "2026-07-02T14:15:53Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:15:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:53Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm.18be7ea5cd833bc8 namespace: kserve-ci-e2e-test resourceVersion: "26102" uid: 297d9c88-b9cb-4372-a772-513edadc0837 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 373c3199-3158-4e4c-8eab-6105fc0d4734 kind: Event lastTimestamp: "2026-07-02T14:15:53Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:15:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:53Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm.18be7ea5cd8e0292 namespace: kserve-ci-e2e-test resourceVersion: "26103" uid: f7eae802-0d76-4694-a855-ad0a40825939 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 373c3199-3158-4e4c-8eab-6105fc0d4734 kind: Event lastTimestamp: "2026-07-02T14:15:53Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:15:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:53Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm.18be7ea5d056abdf namespace: kserve-ci-e2e-test resourceVersion: "26104" uid: f74e2140-4586-46fc-bed4-d93f37290173 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 373c3199-3158-4e4c-8eab-6105fc0d4734 kind: Event lastTimestamp: "2026-07-02T14:15:53Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:15:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:53Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm.18be7ea5d0d10ec6 namespace: kserve-ci-e2e-test resourceVersion: "26105" uid: 7e297ad7-d3a8-4887-8fde-550b6f34bf9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:15:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 373c3199-3158-4e4c-8eab-6105fc0d4734 kind: Event lastTimestamp: "2026-07-02T14:16:29Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:15:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:29Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm.18be7ea5e550ed64 namespace: kserve-ci-e2e-test resourceVersion: "26603" uid: ea0fd233-3d68-421b-a7d0-7e202c761f51 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:33:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 373c3199-3158-4e4c-8eab-6105fc0d4734 kind: Event lastTimestamp: "2026-07-02T14:33:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:33:11Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm.18be7f97a483363c namespace: kserve-ci-e2e-test resourceVersion: "33713" uid: 6b318057-5903-4156-badb-925fe1237d8e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:33:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 373c3199-3158-4e4c-8eab-6105fc0d4734 kind: Event lastTimestamp: "2026-07-02T14:33:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:33:11Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm.18be7f97a483db92 namespace: kserve-ci-e2e-test resourceVersion: "33714" uid: b83bb248-b8f0-4e1c-aeb9-e0c70c67e1da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:33:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm namespace: kserve-ci-e2e-test resourceVersion: "26045" uid: 373c3199-3158-4e4c-8eab-6105fc0d4734 kind: Event lastTimestamp: "2026-07-02T14:33:15Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-02T14:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:33:15Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm.18be7f988196f24c namespace: kserve-ci-e2e-test resourceVersion: "33782" uid: 33b63e05-6035-4cf9-ac8d-4b1ba2d2aa0e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8d2ee-predictor-6f9b7fbf65 namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: 6872052d-d3b1-46c3-8302-14ddbff6a2b2 kind: Event lastTimestamp: "2026-07-02T14:15:51Z" message: 'Created pod: error-404-isvc-8d2ee-predictor-6f9b7fbf65-8r9xm' metadata: creationTimestamp: "2026-07-02T14:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:51Z" name: error-404-isvc-8d2ee-predictor-6f9b7fbf65.18be7ea58acefccc namespace: kserve-ci-e2e-test resourceVersion: "26044" uid: 6457976c-64a2-4461-91b4-50c436ba7e96 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8d2ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26031" uid: 318e6d52-bb51-4787-94b2-b00bc471b667 kind: Event lastTimestamp: "2026-07-02T14:15:51Z" message: Scaled up replica set error-404-isvc-8d2ee-predictor-6f9b7fbf65 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:51Z" name: error-404-isvc-8d2ee-predictor.18be7ea588c52e79 namespace: kserve-ci-e2e-test resourceVersion: "26036" uid: 298c17e2-ce13-4af1-9cdf-2d77296baf6c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d2ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26032" uid: 7dda35d5-6458-468e-a3ee-371b82ca7551 kind: Event lastTimestamp: "2026-07-02T14:16:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:16:21Z" name: error-404-isvc-8d2ee-predictor.18be7ea9074ad106 namespace: kserve-ci-e2e-test resourceVersion: "26542" uid: 7f69ae95-fb3f-426e-a92d-eebecc0efdc5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d2ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26032" uid: 7dda35d5-6458-468e-a3ee-371b82ca7551 kind: Event lastTimestamp: "2026-07-02T14:16:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:16:21Z" name: error-404-isvc-8d2ee-predictor.18be7ea907c13829 namespace: kserve-ci-e2e-test resourceVersion: "26543" uid: 77761a31-d88a-4aa5-b794-c2a3e1017900 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:16:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d2ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: 7dda35d5-6458-468e-a3ee-371b82ca7551 kind: Event lastTimestamp: "2026-07-02T14:17:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:16:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:06Z" name: error-404-isvc-8d2ee-predictor.18be7eb00529030f namespace: kserve-ci-e2e-test resourceVersion: "27002" uid: d4a7ae12-5043-49af-aca3-52c133a5e5ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:16:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d2ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: 7dda35d5-6458-468e-a3ee-371b82ca7551 kind: Event lastTimestamp: "2026-07-02T14:17:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:16:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:06Z" name: error-404-isvc-8d2ee-predictor.18be7eb0058cebfb namespace: kserve-ci-e2e-test resourceVersion: "27003" uid: c7b09950-d15b-4944-83a6-9367798960df reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:15:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d2ee namespace: kserve-ci-e2e-test resourceVersion: "26013" uid: d54b40f6-7beb-4dfd-9486-80514f65192d kind: Event lastTimestamp: "2026-07-02T14:16:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:16:39Z" name: error-404-isvc-8d2ee.18be7ea588c1efda namespace: kserve-ci-e2e-test resourceVersion: "26702" uid: 178dc2db-6757-47da-b016-51eee5b89126 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d2ee namespace: kserve-ci-e2e-test resourceVersion: "26013" uid: d54b40f6-7beb-4dfd-9486-80514f65192d kind: Event lastTimestamp: "2026-07-02T14:15:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-8d2ee": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8d2ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:15:51Z" name: error-404-isvc-8d2ee.18be7ea589230b52 namespace: kserve-ci-e2e-test resourceVersion: "26039" uid: 3c2b58c6-3232-4aaa-a195-57116c7085f1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d2ee namespace: kserve-ci-e2e-test resourceVersion: "26013" uid: d54b40f6-7beb-4dfd-9486-80514f65192d kind: Event lastTimestamp: "2026-07-02T14:15:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8d2ee": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:15:51Z" name: error-404-isvc-8d2ee.18be7ea589236614 namespace: kserve-ci-e2e-test resourceVersion: "26041" uid: 635c5d00-9fc9-4e48-980c-6b098a641a10 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d2ee namespace: kserve-ci-e2e-test resourceVersion: "26700" uid: d54b40f6-7beb-4dfd-9486-80514f65192d kind: Event lastTimestamp: "2026-07-02T14:16:39Z" message: InferenceService [error-404-isvc-8d2ee] is Ready metadata: creationTimestamp: "2026-07-02T14:16:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:16:39Z" name: error-404-isvc-8d2ee.18be7eb0a4019e79 namespace: kserve-ci-e2e-test resourceVersion: "26701" uid: 7fb47e77-326b-43cd-96e0-3d0c78c7ec3e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:05:26.866469Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx namespace: kserve-ci-e2e-test resourceVersion: "20523" uid: e1ed2cb5-74ad-404f-a6e4-68b4432279b3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c559a-predictor-7c98468fff-9t7fx to ip-10-0-133-209.ec2.internal metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:05:26Z" name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx.18be7e1400ff640f namespace: kserve-ci-e2e-test resourceVersion: "20527" uid: 1df02c6d-1732-4d7c-979c-496b498d0a71 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx namespace: kserve-ci-e2e-test resourceVersion: "20533" uid: e1ed2cb5-74ad-404f-a6e4-68b4432279b3 kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:05:27Z" name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx.18be7e141a9add72 namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: 0fe9df0d-4460-44ee-bf8e-4c741fa54d14 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: e1ed2cb5-74ad-404f-a6e4-68b4432279b3 kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:27Z" name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx.18be7e141c1b83ee namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 7356623c-6ff6-444f-8252-093f0a2caf1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: e1ed2cb5-74ad-404f-a6e4-68b4432279b3 kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:27Z" name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx.18be7e141fceb583 namespace: kserve-ci-e2e-test resourceVersion: "20552" uid: bcf6c681-2ba8-4035-a090-9e801f40020d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: e1ed2cb5-74ad-404f-a6e4-68b4432279b3 kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:27Z" name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx.18be7e142049f4f3 namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: eacd9cca-1852-49b4-85d6-2827c051d613 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: e1ed2cb5-74ad-404f-a6e4-68b4432279b3 kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:27Z" name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx.18be7e142054d538 namespace: kserve-ci-e2e-test resourceVersion: "20554" uid: c6af6f3a-bcfb-4e74-b972-21aad0de6c2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: e1ed2cb5-74ad-404f-a6e4-68b4432279b3 kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:27Z" name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx.18be7e14241d50fd namespace: kserve-ci-e2e-test resourceVersion: "20555" uid: 7962f75d-f126-4588-85c1-b570d07b14a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: e1ed2cb5-74ad-404f-a6e4-68b4432279b3 kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:27Z" name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx.18be7e14249d56f2 namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: c5e27019-8d48-408d-b657-f5f66bf02864 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:05:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: e1ed2cb5-74ad-404f-a6e4-68b4432279b3 kind: Event lastTimestamp: "2026-07-02T14:06:04Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:05:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:04Z" name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx.18be7e144a4a684f namespace: kserve-ci-e2e-test resourceVersion: "21072" uid: 7d2c06e6-b309-4191-9c16-6ef425276327 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: e1ed2cb5-74ad-404f-a6e4-68b4432279b3 kind: Event lastTimestamp: "2026-07-02T14:06:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:36Z" name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx.18be7e2452b7db95 namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: 0bdb7223-b068-493c-8fc3-595c6fb6892b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: e1ed2cb5-74ad-404f-a6e4-68b4432279b3 kind: Event lastTimestamp: "2026-07-02T14:06:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:36Z" name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx.18be7e2452b8c055 namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: a3f85a4e-1fb9-492f-a5b0-965ad4bcced8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: e1ed2cb5-74ad-404f-a6e4-68b4432279b3 kind: Event lastTimestamp: "2026-07-02T14:06:39Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:39Z" name: error-404-isvc-c559a-predictor-7c98468fff-9t7fx.18be7e24d26cf24c namespace: kserve-ci-e2e-test resourceVersion: "21547" uid: 3dbb59d1-27c7-40cd-8888-fbc591914e19 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c559a-predictor-7c98468fff namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: d5493736-8362-48cc-a56e-5a3dae57f9dd kind: Event lastTimestamp: "2026-07-02T14:05:26Z" message: 'Created pod: error-404-isvc-c559a-predictor-7c98468fff-9t7fx' metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:26Z" name: error-404-isvc-c559a-predictor-7c98468fff.18be7e140090cf04 namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 81448f04-63d8-46be-b14f-6f3fd11611b0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c559a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20513" uid: b6050d8d-60d9-4e5f-a8e5-ccbd297638ff kind: Event lastTimestamp: "2026-07-02T14:05:26Z" message: Scaled up replica set error-404-isvc-c559a-predictor-7c98468fff from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:26Z" name: error-404-isvc-c559a-predictor.18be7e13feff7eff namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: e13af0da-39e6-415f-8a8b-85d196f24e4c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c559a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 7b1e7a0a-25d7-45a4-a112-98c24543b4b6 kind: Event lastTimestamp: "2026-07-02T14:05:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:41Z" name: error-404-isvc-c559a-predictor.18be7e177d8eebab namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: 45df81c7-c898-4ef3-a583-a85836a4b870 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c559a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 7b1e7a0a-25d7-45a4-a112-98c24543b4b6 kind: Event lastTimestamp: "2026-07-02T14:05:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:41Z" name: error-404-isvc-c559a-predictor.18be7e177e0f5618 namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 7db5e571-0d99-460b-9eae-95eea7fd5604 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:05:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c559a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 7b1e7a0a-25d7-45a4-a112-98c24543b4b6 kind: Event lastTimestamp: "2026-07-02T14:06:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:26Z" name: error-404-isvc-c559a-predictor.18be7e1afcd01e3b namespace: kserve-ci-e2e-test resourceVersion: "21291" uid: 96c8ec73-1313-4750-a501-ab2ab2c9c162 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:05:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c559a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 7b1e7a0a-25d7-45a4-a112-98c24543b4b6 kind: Event lastTimestamp: "2026-07-02T14:06:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:05:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:26Z" name: error-404-isvc-c559a-predictor.18be7e1afd3ac594 namespace: kserve-ci-e2e-test resourceVersion: "21292" uid: ce6a7c55-af18-4f83-a4db-09400023a812 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c559a namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: 41376a63-3fca-49a3-a6f2-3b41b9c916ba kind: Event lastTimestamp: "2026-07-02T14:06:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:14Z" name: error-404-isvc-c559a.18be7e13ff04f018 namespace: kserve-ci-e2e-test resourceVersion: "21168" uid: a9dab619-ccb9-43b0-92cd-47e773b0bae8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c559a namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: 41376a63-3fca-49a3-a6f2-3b41b9c916ba kind: Event lastTimestamp: "2026-07-02T14:05:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-c559a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c559a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:26Z" name: error-404-isvc-c559a.18be7e13ff602823 namespace: kserve-ci-e2e-test resourceVersion: "20520" uid: 766946dd-3e8d-41cf-ac6d-5660892ebbaf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c559a namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: 41376a63-3fca-49a3-a6f2-3b41b9c916ba kind: Event lastTimestamp: "2026-07-02T14:05:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c559a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:26Z" name: error-404-isvc-c559a.18be7e13ff60659b namespace: kserve-ci-e2e-test resourceVersion: "20521" uid: 1117bbfa-cc47-4910-bb54-2f946cfc2057 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c559a namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 41376a63-3fca-49a3-a6f2-3b41b9c916ba kind: Event lastTimestamp: "2026-07-02T14:06:14Z" message: InferenceService [error-404-isvc-c559a] is Ready metadata: creationTimestamp: "2026-07-02T14:06:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:14Z" name: error-404-isvc-c559a.18be7e1f05bf55e4 namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: 8848cc3e-8979-4045-a0f6-3c063c7b4f21 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:04:30.661379Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 6bd57168-96c6-4681-8758-f1026962e078 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c675b-predictor-5547c66cc5-ztplz to ip-10-0-133-209.ec2.internal metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:04:30Z" name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz.18be7e06eae9c1eb namespace: kserve-ci-e2e-test resourceVersion: "19933" uid: 6bc11cb0-2dd4-489f-bb67-ccd22bf56f60 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 6bd57168-96c6-4681-8758-f1026962e078 kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c675b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:30Z" name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz.18be7e06f38db82a namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: f1194601-623e-4289-b4fe-0e4abeca39b3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 6bd57168-96c6-4681-8758-f1026962e078 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:04:31Z" name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz.18be7e07284da726 namespace: kserve-ci-e2e-test resourceVersion: "19974" uid: a296e3d7-7913-41e7-963e-cd8dd47b983c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 6bd57168-96c6-4681-8758-f1026962e078 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:31Z" name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz.18be7e0729a50b81 namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: 8907432a-1670-4afa-96d4-655476030769 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 6bd57168-96c6-4681-8758-f1026962e078 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:31Z" name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz.18be7e072c9ce2e6 namespace: kserve-ci-e2e-test resourceVersion: "19980" uid: 63774edc-167a-44b0-94f9-fc9941079a96 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 6bd57168-96c6-4681-8758-f1026962e078 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:31Z" name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz.18be7e072d2f138f namespace: kserve-ci-e2e-test resourceVersion: "19981" uid: 10230a02-0af2-4455-90a0-4a0d0fa5038a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 6bd57168-96c6-4681-8758-f1026962e078 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:31Z" name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz.18be7e072d38f14d namespace: kserve-ci-e2e-test resourceVersion: "19983" uid: fb89bf08-c67c-41b7-9416-5f63036ff6a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 6bd57168-96c6-4681-8758-f1026962e078 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:31Z" name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz.18be7e072ffffa92 namespace: kserve-ci-e2e-test resourceVersion: "19986" uid: 1de69dfc-974e-4bee-a54c-dab9abae7dff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 6bd57168-96c6-4681-8758-f1026962e078 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:31Z" name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz.18be7e07307683ed namespace: kserve-ci-e2e-test resourceVersion: "19987" uid: 3e00c02c-12de-4624-b77f-59f5490b112b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 6bd57168-96c6-4681-8758-f1026962e078 kind: Event lastTimestamp: "2026-07-02T14:05:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:07Z" name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz.18be7e0733d8e0e7 namespace: kserve-ci-e2e-test resourceVersion: "20272" uid: 7b5ec38c-b013-40ae-aadb-b1452e8a6ea6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 6bd57168-96c6-4681-8758-f1026962e078 kind: Event lastTimestamp: "2026-07-02T14:05:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:40Z" name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz.18be7e173e79dc7d namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 58238b55-c732-42c1-aac1-dc5e33b114c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 6bd57168-96c6-4681-8758-f1026962e078 kind: Event lastTimestamp: "2026-07-02T14:05:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:40Z" name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz.18be7e173e7a8f19 namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: ed766712-1dec-450b-a133-b92c9aa4944d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 6bd57168-96c6-4681-8758-f1026962e078 kind: Event lastTimestamp: "2026-07-02T14:05:42Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:42Z" name: error-404-isvc-c675b-predictor-5547c66cc5-ztplz.18be7e17bbf1a47f namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: 3ae66807-f41f-456c-a427-2d32e17bc5e3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-209.ec2.internal source: component: kubelet host: ip-10-0-133-209.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c675b-predictor-5547c66cc5 namespace: kserve-ci-e2e-test resourceVersion: "19918" uid: 119b56e0-96a7-4693-b509-adc75fd67493 kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: 'Created pod: error-404-isvc-c675b-predictor-5547c66cc5-ztplz' metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:04:30Z" name: error-404-isvc-c675b-predictor-5547c66cc5.18be7e06ea6e7c7e namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: a95282be-bb49-46cb-aca6-141ec056293a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c675b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: 5736dc98-d34a-44a0-a1fb-330426e2e569 kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: Scaled up replica set error-404-isvc-c675b-predictor-5547c66cc5 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:04:30Z" name: error-404-isvc-c675b-predictor.18be7e06e8c4512f namespace: kserve-ci-e2e-test resourceVersion: "19921" uid: f85dd407-2ac2-45c5-9b54-57436b7b3ae0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:04:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c675b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: 84b725f0-f0e7-4b15-b88e-4ba4b7e717ed kind: Event lastTimestamp: "2026-07-02T14:05:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:00Z" name: error-404-isvc-c675b-predictor.18be7e0a67880f18 namespace: kserve-ci-e2e-test resourceVersion: "20225" uid: a1b36ee5-ed85-4991-b5a5-9419857c132b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:04:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c675b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: 84b725f0-f0e7-4b15-b88e-4ba4b7e717ed kind: Event lastTimestamp: "2026-07-02T14:05:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:00Z" name: error-404-isvc-c675b-predictor.18be7e0a67db9d3c namespace: kserve-ci-e2e-test resourceVersion: "20226" uid: 54a750f5-0cb9-4431-9c74-10e4a5eed93d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c675b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: 84b725f0-f0e7-4b15-b88e-4ba4b7e717ed kind: Event lastTimestamp: "2026-07-02T14:05:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:30Z" name: error-404-isvc-c675b-predictor.18be7e11655ad2f6 namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: 1d8fa1db-fcb3-4012-89f7-f21f904b0f5d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c675b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20108" uid: 84b725f0-f0e7-4b15-b88e-4ba4b7e717ed kind: Event lastTimestamp: "2026-07-02T14:05:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:30Z" name: error-404-isvc-c675b-predictor.18be7e11659fe500 namespace: kserve-ci-e2e-test resourceVersion: "20628" uid: 340e4269-f3a9-46cd-9794-d34130ae9e38 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c675b namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: 7a391c6a-9bba-4f41-aec4-82f951a65f5f kind: Event lastTimestamp: "2026-07-02T14:05:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:18Z" name: error-404-isvc-c675b.18be7e06e8c779b8 namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: dbe46f72-7a2d-43e6-a711-7d96f53fe59a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c675b namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: 7a391c6a-9bba-4f41-aec4-82f951a65f5f kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: 'Failed to update status for InferenceService "error-404-isvc-c675b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c675b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:04:30Z" name: error-404-isvc-c675b.18be7e06e945f1cb namespace: kserve-ci-e2e-test resourceVersion: "19923" uid: 10444bae-0590-46d5-a06f-19d28d14d873 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c675b namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: 7a391c6a-9bba-4f41-aec4-82f951a65f5f kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c675b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:04:30Z" name: error-404-isvc-c675b.18be7e06e9464315 namespace: kserve-ci-e2e-test resourceVersion: "19925" uid: 82a3bcbc-423a-44dc-9301-5bac9da8dea3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c675b namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: 7a391c6a-9bba-4f41-aec4-82f951a65f5f kind: Event lastTimestamp: "2026-07-02T14:05:17Z" message: InferenceService [error-404-isvc-c675b] is Ready metadata: creationTimestamp: "2026-07-02T14:05:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:17Z" name: error-404-isvc-c675b.18be7e11ef2900cc namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 67007a11-d2a3-4e78-b5e3-2c5fbb217f42 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:53:41.692506Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14443" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-866d764d98-72g5f to ip-10-0-137-40.ec2.internal metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:53:41Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d6fd15aa481 namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: 1af3b0c8-1a04-4ffe-b911-9b80bb493012 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:53:42Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:53:42Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d6fecb9e60a namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: fb85b8b3-a162-44cc-bc5d-033432f51fc2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:53:42Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:42Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d6feedd423b namespace: kserve-ci-e2e-test resourceVersion: "14539" uid: 224cbda0-0bdc-4572-a17a-44feb76ca61d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:53:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:42Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d6ff244f5fe namespace: kserve-ci-e2e-test resourceVersion: "14547" uid: 62eaec42-fc20-44b6-bd17-cc3c421a7b1c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:53:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:42Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d6ff2df192f namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: aae48526-de19-4866-a763-5c4c5aa1946f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:53:46Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1695" metadata: creationTimestamp: "2026-07-02T13:53:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:46Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d70ff9218cc namespace: kserve-ci-e2e-test resourceVersion: "14624" uid: cb05baad-253d-4363-a555-e16f9a79b11f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1695" in 28.62s (28.62s including waiting). Image size: 1555877318 bytes.' metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d77a979c67a namespace: kserve-ci-e2e-test resourceVersion: "14880" uid: bae7d454-ade0-49cc-b4a0-dc864b4eb69c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d77b087981e namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: 78d82829-bed3-45d7-8f3b-53e25052172a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d77b1d8fa74 namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 61827dea-2ea7-4ba5-9497-f86103f6b554 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d77b1e4eeea namespace: kserve-ci-e2e-test resourceVersion: "14887" uid: 4ba1101f-1c06-419d-a6a8-ff058e31e420 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:54:18Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.079s (3.079s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T13:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:18Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d78696c0582 namespace: kserve-ci-e2e-test resourceVersion: "14914" uid: d64285e9-25cd-4651-be6a-9e196820b67c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:54:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:18Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d786dd6a70d namespace: kserve-ci-e2e-test resourceVersion: "14918" uid: f24d6099-95c1-4872-80e1-acb195cc7908 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:54:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:18Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d786e53ac42 namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 36a563cc-8639-4e8a-a3e6-f50e116b2a98 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:54:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:56:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:54:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:16Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d78bd2525a1 namespace: kserve-ci-e2e-test resourceVersion: "16353" uid: fe8f9e6b-61af-4a7e-bc57-a9a84eabdb25 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:56:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:11Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d92cadccef7 namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: ddd28af5-e015-4880-b00d-73497f8eac39 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:56:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:11Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d92cadd4506 namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: dff10111-f1ff-4ded-93b2-147af1686370 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 6d4dc493-0eab-44a2-b0f9-9cf16928015d kind: Event lastTimestamp: "2026-07-02T13:56:16Z" message: 'Readiness probe failed: Get "https://10.132.0.27:8643/healthz": dial tcp 10.132.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:16Z" name: isvc-sklearn-graph-1-predictor-866d764d98-72g5f.18be7d93bf7e9896 namespace: kserve-ci-e2e-test resourceVersion: "16352" uid: 66927404-c456-4c4a-bfd3-b38611b06aa8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-866d764d98 namespace: kserve-ci-e2e-test resourceVersion: "14434" uid: a45e180a-e64e-44d6-b56e-bf208890a5e9 kind: Event lastTimestamp: "2026-07-02T13:53:41Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-866d764d98-72g5f' metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:41Z" name: isvc-sklearn-graph-1-predictor-866d764d98.18be7d6fd0d45775 namespace: kserve-ci-e2e-test resourceVersion: "14445" uid: 40a2b88b-f828-47e9-a04c-fe5acc3fe13e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14432" uid: 4a63ce2f-c5fd-48e6-aaef-c4943bc6e135 kind: Event lastTimestamp: "2026-07-02T13:53:41Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-866d764d98 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:41Z" name: isvc-sklearn-graph-1-predictor.18be7d6fcdd5069e namespace: kserve-ci-e2e-test resourceVersion: "14435" uid: 2849a757-03f2-43a5-b0d4-27b17eba4909 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: 052abd9e-7b98-49ba-b9f8-97cb84ccc0bb kind: Event lastTimestamp: "2026-07-02T13:54:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:54:26Z" name: isvc-sklearn-graph-1-predictor.18be7d734c85eba5 namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: f5a3f61c-69df-41d0-b8a2-af387cbe9ec1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14433" uid: 052abd9e-7b98-49ba-b9f8-97cb84ccc0bb kind: Event lastTimestamp: "2026-07-02T13:54:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:54:26Z" name: isvc-sklearn-graph-1-predictor.18be7d734cf50175 namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: c1bbf867-6306-4cd0-b0b7-670a128fd2f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:54:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14716" uid: 052abd9e-7b98-49ba-b9f8-97cb84ccc0bb kind: Event lastTimestamp: "2026-07-02T13:55:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:56Z" name: isvc-sklearn-graph-1-predictor.18be7d7dc92b75ec namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: 5a370a2d-c313-44a7-8a98-c2539b5e2cc7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:54:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14716" uid: 052abd9e-7b98-49ba-b9f8-97cb84ccc0bb kind: Event lastTimestamp: "2026-07-02T13:55:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:56Z" name: isvc-sklearn-graph-1-predictor.18be7d7dc9a664be namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 6c25b399-6604-4462-b1a5-a992b422031e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: 54f9a26c-d304-443e-9a49-a0353c7c52bb kind: Event lastTimestamp: "2026-07-02T13:55:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:36Z" name: isvc-sklearn-graph-1.18be7d6fcdd99cc5 namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: 099bc5f7-3b9a-44bd-b121-5d00813148ed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15784" uid: 54f9a26c-d304-443e-9a49-a0353c7c52bb kind: Event lastTimestamp: "2026-07-02T13:55:36Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-02T13:55:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:36Z" name: isvc-sklearn-graph-1.18be7d8a7a212981 namespace: kserve-ci-e2e-test resourceVersion: "15786" uid: a1249c5e-fc18-478f-b13e-478d67dbbbb8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:53:42.106373Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14526" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw to ip-10-0-137-40.ec2.internal metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:53:42Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d6fea05c1ff namespace: kserve-ci-e2e-test resourceVersion: "14531" uid: 5bfe4725-8858-4f65-9d43-20589f66ab79 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:53:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:42Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d6ff22bcde5 namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: 69b2dfc7-4b3f-4f16-901d-0868cf433ab3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:43Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14536" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:53:43Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:53:43Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d7027ab0b20 namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: ebcba680-7b66-485f-aace-8862a3339beb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:53:43Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T13:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:43Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d70297da0ab namespace: kserve-ci-e2e-test resourceVersion: "14586" uid: 83fd7683-5739-4ef8-8a26-ffaeb0c6188d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:53:43Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:43Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d702c60a18e namespace: kserve-ci-e2e-test resourceVersion: "14587" uid: 29c6bec5-6b71-474c-aaf2-c863c9383cbc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:43Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:53:43Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:53:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:43Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d702cd3a0d6 namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: d424518d-a014-46b9-a5cb-a0a328c0866f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:53:47Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1695" metadata: creationTimestamp: "2026-07-02T13:53:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:47Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d713c5d52f2 namespace: kserve-ci-e2e-test resourceVersion: "14637" uid: e2069b9a-c77d-4269-89be-267bf8de1782 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1695" in 27.6s (27.6s including waiting). Image size: 1555877318 bytes.' metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d77a97d89ef namespace: kserve-ci-e2e-test resourceVersion: "14881" uid: f07cd00a-5285-43ff-9b4f-50d7a4e2b792 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d77af314f8a namespace: kserve-ci-e2e-test resourceVersion: "14882" uid: d27eed95-0652-4c5c-9e07-433775322aa5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d77b0866299 namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: 7c323604-ab8c-44b7-b487-04fd28e75824 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d77b092df91 namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: 77e6b810-0791-4640-8c69-ac6b77756836 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:54:18Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.112s (3.112s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T13:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:18Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d786a166ec5 namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 5ac2c317-583f-4171-be35-10a7d506c93e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:54:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:18Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d786dcbc778 namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: a9c87a84-81ad-4a2f-9e0f-b878071cc259 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:54:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:18Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d786e494b81 namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: ebf993ad-1273-44e4-ac6e-a7692162760d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-02T13:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:56:15Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:15Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d788128aaa4 namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: ee602b84-66c3-4393-97e8-c4ef02dc2038 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:56:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:11Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d92c6c6aa96 namespace: kserve-ci-e2e-test resourceVersion: "16168" uid: 0172b75c-3c1a-4acf-a6ac-c370dd1f6e6a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:56:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:11Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d92c6c76654 namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: b5c72242-a9e3-458a-8f3c-2bc76df2115e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw namespace: kserve-ci-e2e-test resourceVersion: "14528" uid: da6457ff-394d-4ad1-9f8f-a0e9e880dcce kind: Event lastTimestamp: "2026-07-02T13:56:15Z" message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp 10.132.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:15Z" name: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw.18be7d9383a7498e namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 67d791bd-32f5-4be1-b677-4956c5de4bac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-555c647c7d namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 9915ffe7-8fda-4cc1-9232-3bc1c86b4956 kind: Event lastTimestamp: "2026-07-02T13:53:42Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-555c647c7d-2z4lw' metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:42Z" name: isvc-sklearn-graph-2-predictor-555c647c7d.18be7d6fe9527e28 namespace: kserve-ci-e2e-test resourceVersion: "14527" uid: 4b19fbb8-372a-4b55-a2bd-7e5887c4f491 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14514" uid: c6fb508a-11c7-4a74-87ff-0546b84f3166 kind: Event lastTimestamp: "2026-07-02T13:53:42Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-555c647c7d from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:42Z" name: isvc-sklearn-graph-2-predictor.18be7d6fe51de16d namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: acc274e1-6e0e-4a2d-88c9-967dd8085e49 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:53:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14516" uid: dba072ed-d71c-45ad-8f40-20c9377ea9f0 kind: Event lastTimestamp: "2026-07-02T13:54:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:54:42Z" name: isvc-sklearn-graph-2-predictor.18be7d736363b536 namespace: kserve-ci-e2e-test resourceVersion: "15126" uid: 3d0742a6-d6e9-4fee-a981-c1926c7ff856 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:53:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14516" uid: dba072ed-d71c-45ad-8f40-20c9377ea9f0 kind: Event lastTimestamp: "2026-07-02T13:54:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:53:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:54:42Z" name: isvc-sklearn-graph-2-predictor.18be7d7363d68788 namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: b3efb878-de91-41ff-8195-01d0a7472611 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:54:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: dba072ed-d71c-45ad-8f40-20c9377ea9f0 kind: Event lastTimestamp: "2026-07-02T13:55:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:57Z" name: isvc-sklearn-graph-2-predictor.18be7d815f6df2d2 namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 0e3bc074-2bf1-42e2-a6a1-9975fb8c285c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:54:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: dba072ed-d71c-45ad-8f40-20c9377ea9f0 kind: Event lastTimestamp: "2026-07-02T13:55:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:54:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:57Z" name: isvc-sklearn-graph-2-predictor.18be7d815fd25aa9 namespace: kserve-ci-e2e-test resourceVersion: "15959" uid: 59b7373b-6d2d-44a1-9b2f-3a337e1db190 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14499" uid: 4523041d-beed-43c7-bbab-b25f6dda6ccd kind: Event lastTimestamp: "2026-07-02T13:55:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:35Z" name: isvc-sklearn-graph-2.18be7d6fe4b8e0c7 namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: e2eafeee-059d-41f1-bea3-c46022f8e951 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 4523041d-beed-43c7-bbab-b25f6dda6ccd kind: Event lastTimestamp: "2026-07-02T13:55:35Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-02T13:55:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:35Z" name: isvc-sklearn-graph-2.18be7d8a39c14eb6 namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: b1bbc6de-75f8-4f93-a477-c571418cefbf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:53:41.964957Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-72pkg to ip-10-0-137-40.ec2.internal metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:53:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d6fe197f07e namespace: kserve-ci-e2e-test resourceVersion: "14505" uid: 0fc0fb58-d07b-4577-91a2-26017b76073f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14519" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:53:42Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:53:42Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d6ffc804e7e namespace: kserve-ci-e2e-test resourceVersion: "14556" uid: cdc700e8-c02e-41b4-a2ee-cd340de1edde reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:53:42Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:cd2b86b0a2c9500aabe87a633c3b9bab30a880b19ff42859e99cc17f113ecdb0" already present on machine metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:42Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d6ffe1e8285 namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: b54b8008-defa-41ef-af62-fff5194faa66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:53:42Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:42Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d70018d58fc namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: f94b7115-a663-4ff0-a2ae-5a0fa033cca3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:42Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:53:42Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T13:53:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:42Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d700208aa3c namespace: kserve-ci-e2e-test resourceVersion: "14565" uid: 7d81c4ba-f4e3-46e1-bb5e-edfd1d1d5723 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:53:46Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-02T13:53:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:46Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d70ff7dffe1 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 4f71ff5e-2e38-4d28-b363-0ea7949ee949 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 29.093s (29.093s including waiting). Image size: 1305293048 bytes.' metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d77c59c028a namespace: kserve-ci-e2e-test resourceVersion: "14889" uid: 17c1bdcf-06af-4833-8b95-c28f58561ae9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d77caf7a213 namespace: kserve-ci-e2e-test resourceVersion: "14891" uid: 9ccdf31b-c30e-456d-b78a-f2e57c7a0dab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d77cc5fcbb6 namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: 1bf4c44e-d3da-463d-9574-0a1847191846 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:54:15Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T13:54:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d77cc6c8626 namespace: kserve-ci-e2e-test resourceVersion: "14893" uid: a606f449-9552-425d-a13c-47f87d871843 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:54:18Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.634s (2.634s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T13:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d786971e8bd namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: e1f0b978-86b6-4495-85d2-6297dc88a4d3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:54:18Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d786dd7c4b2 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 9305dee8-80b4-4443-a595-09d9ba083249 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:54:18Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:54:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d786e604314 namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: dc2f6e3c-9b8d-4a16-b5d3-956577ecedff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T13:54:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:56:15Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:54:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d78811047c2 namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: 3a02d2ea-6caf-4a24-a118-68a58d9de986 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:56:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d92d36f14f4 namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: 637e3a9b-b3d6-4b5d-9344-4f7f4f1fb138 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:56:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d92d3704d6b namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: 896f99b2-8644-45c1-9fff-494efab3a8b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-72pkg namespace: kserve-ci-e2e-test resourceVersion: "14503" uid: d724e45d-5e3c-4ac9-bc2d-891d8da92e1b kind: Event lastTimestamp: "2026-07-02T13:56:15Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:15Z" name: isvc-xgboost-graph-predictor-555fc69d56-72pkg.18be7d9383a7a738 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: ac4a9c5d-6c4d-49a3-b92a-1741228891d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-40.ec2.internal source: component: kubelet host: ip-10-0-137-40.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14489" uid: 078455de-31f0-453c-980b-4805c9d0b0b6 kind: Event lastTimestamp: "2026-07-02T13:53:41Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-72pkg' metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:41Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be7d6fe12409a3 namespace: kserve-ci-e2e-test resourceVersion: "14502" uid: 2adaa09e-da80-4c6e-ba39-88151d4a2858 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14487" uid: 9fbc3c8f-b2ef-41d1-87aa-5f30504b8a3e kind: Event lastTimestamp: "2026-07-02T13:53:41Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:41Z" name: isvc-xgboost-graph-predictor.18be7d6fdb98fbec namespace: kserve-ci-e2e-test resourceVersion: "14490" uid: 5c328b0a-46d9-486a-b8d9-afc98918c126 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: e298beb6-e9c2-4a38-9bc3-668c553bb2fe kind: Event lastTimestamp: "2026-07-02T13:54:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:54:56Z" name: isvc-xgboost-graph-predictor.18be7d735a77de28 namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: b8184cae-fec7-4007-815f-e448b870b575 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14488" uid: e298beb6-e9c2-4a38-9bc3-668c553bb2fe kind: Event lastTimestamp: "2026-07-02T13:54:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:54:56Z" name: isvc-xgboost-graph-predictor.18be7d735ad99daa namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: 34703115-a2a5-4b8c-85eb-1abebfe0c7a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:55:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: e298beb6-e9c2-4a38-9bc3-668c553bb2fe kind: Event lastTimestamp: "2026-07-02T13:55:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:56Z" name: isvc-xgboost-graph-predictor.18be7d84d4a5d8ac namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: 1442dcca-cfe5-4d57-9a4e-44b5a35803f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:55:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14730" uid: e298beb6-e9c2-4a38-9bc3-668c553bb2fe kind: Event lastTimestamp: "2026-07-02T13:55:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:55:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:56Z" name: isvc-xgboost-graph-predictor.18be7d84d513c19e namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: cf5a9ebf-98ae-4624-950c-eddb8e034f43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: 0abf4262-ed20-4e6e-a24f-b1553f8db9b7 kind: Event lastTimestamp: "2026-07-02T13:55:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:25Z" name: isvc-xgboost-graph.18be7d6fdba7778d namespace: kserve-ci-e2e-test resourceVersion: "15683" uid: cc7ef11f-6b53-42f6-bcca-342eb9beef6a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: 0abf4262-ed20-4e6e-a24f-b1553f8db9b7 kind: Event lastTimestamp: "2026-07-02T13:55:25Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-02T13:55:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:25Z" name: isvc-xgboost-graph.18be7d87e60dadec namespace: kserve-ci-e2e-test resourceVersion: "15681" uid: 1647506f-ce33-4284-932d-674ec5c18e9c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:56:01.549184Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-59c85f56c9-2z4kl namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 824b6be7-7681-43c0-9268-95b4cfca312e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-59c85f56c9-2z4kl to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T13:56:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:56:01Z" name: model-chainer-59c85f56c9-2z4kl.18be7d90617630fd namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: b3cf3a7a-fff8-410d-a29a-159351634845 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:01Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-59c85f56c9-2z4kl namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 824b6be7-7681-43c0-9268-95b4cfca312e kind: Event lastTimestamp: "2026-07-02T13:56:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:01Z" name: model-chainer-59c85f56c9-2z4kl.18be7d906a92ee4a namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: ed9235f7-9f4c-4380-9d1b-f38a7a7712dc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:02Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-59c85f56c9-2z4kl namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 824b6be7-7681-43c0-9268-95b4cfca312e kind: Event lastTimestamp: "2026-07-02T13:56:02Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:56:02Z" name: model-chainer-59c85f56c9-2z4kl.18be7d909ea43e48 namespace: kserve-ci-e2e-test resourceVersion: "16027" uid: 78b13de2-db91-4c9b-a7d7-7b1f0b6bc6ab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-59c85f56c9-2z4kl namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 824b6be7-7681-43c0-9268-95b4cfca312e kind: Event lastTimestamp: "2026-07-02T13:56:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" already present on machine metadata: creationTimestamp: "2026-07-02T13:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:02Z" name: model-chainer-59c85f56c9-2z4kl.18be7d909ff839e2 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 19be463a-b814-46fa-bdd1-438f7793f79e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-59c85f56c9-2z4kl namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 824b6be7-7681-43c0-9268-95b4cfca312e kind: Event lastTimestamp: "2026-07-02T13:56:02Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-02T13:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:02Z" name: model-chainer-59c85f56c9-2z4kl.18be7d90a2b1ef4c namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: ed868476-96a1-4e0f-ba4b-3192a8600408 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-59c85f56c9-2z4kl namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 824b6be7-7681-43c0-9268-95b4cfca312e kind: Event lastTimestamp: "2026-07-02T13:56:02Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-02T13:56:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:02Z" name: model-chainer-59c85f56c9-2z4kl.18be7d90a3138fd1 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 3f0cfbfe-6ef5-4b6d-aefa-452f0dd72d5e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-59c85f56c9-2z4kl namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 824b6be7-7681-43c0-9268-95b4cfca312e kind: Event lastTimestamp: "2026-07-02T13:56:11Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:11Z" name: model-chainer-59c85f56c9-2z4kl.18be7d92bea2cace namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: 2df1959f-868a-4c7e-b962-c2b08845e31e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:56:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-59c85f56c9-2z4kl namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: 824b6be7-7681-43c0-9268-95b4cfca312e kind: Event lastTimestamp: "2026-07-02T13:56:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:39Z" name: model-chainer-59c85f56c9-2z4kl.18be7d935f410d6f namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: caf637de-1d0d-4386-84ac-31fe204c9a52 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-59c85f56c9 namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: 3b94da8e-68c4-496e-a496-2edb95b1cc33 kind: Event lastTimestamp: "2026-07-02T13:56:01Z" message: 'Created pod: model-chainer-59c85f56c9-2z4kl' metadata: creationTimestamp: "2026-07-02T13:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:01Z" name: model-chainer-59c85f56c9.18be7d9060b83d9d namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: cf69b8a3-ca9a-4e9d-a039-ade136fb78d7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 0a638907-5a7a-432d-ae17-98fae0aa56c5 kind: Event lastTimestamp: "2026-07-02T13:56:01Z" message: Scaled up replica set model-chainer-59c85f56c9 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:01Z" name: model-chainer.18be7d905f96f2c0 namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: af5077fb-3ae5-41bb-a2c0-7f4b13d4e5d6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: c8c91d75-66a2-4c03-a3c4-ca9a6f93bf5b kind: Event lastTimestamp: "2026-07-02T13:56:01Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-02T13:56:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:56:01Z" name: model-chainer.18be7d9066264197 namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: cd455c40-c6ce-417b-88fb-3f88f364a8f1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:07:37.106343Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6d218-57587c6648-hpj45 namespace: kserve-ci-e2e-test resourceVersion: "22076" uid: 8d9d8706-f755-42c1-9fb7-8c31e211345d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-6d218-57587c6648-hpj45 to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:07:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:07:37Z" name: sequence-graph-6d218-57587c6648-hpj45.18be7e3253e61f1a namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: bede8e03-5235-4422-a90b-c4e05b020a90 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:37Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-6d218-57587c6648-hpj45 namespace: kserve-ci-e2e-test resourceVersion: "22085" uid: 8d9d8706-f755-42c1-9fb7-8c31e211345d kind: Event lastTimestamp: "2026-07-02T14:07:37Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:07:37Z" name: sequence-graph-6d218-57587c6648-hpj45.18be7e326d9c666a namespace: kserve-ci-e2e-test resourceVersion: "22098" uid: 46ad98e3-3972-41b7-92ea-a9f50f3532d1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6d218} kind: Pod name: sequence-graph-6d218-57587c6648-hpj45 namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: 8d9d8706-f755-42c1-9fb7-8c31e211345d kind: Event lastTimestamp: "2026-07-02T14:07:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" already present on machine metadata: creationTimestamp: "2026-07-02T14:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:37Z" name: sequence-graph-6d218-57587c6648-hpj45.18be7e326ee7a53b namespace: kserve-ci-e2e-test resourceVersion: "22101" uid: fb95f5ce-4249-476a-a1f1-c874c40e1540 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6d218} kind: Pod name: sequence-graph-6d218-57587c6648-hpj45 namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: 8d9d8706-f755-42c1-9fb7-8c31e211345d kind: Event lastTimestamp: "2026-07-02T14:07:37Z" message: 'Created container: sequence-graph-6d218' metadata: creationTimestamp: "2026-07-02T14:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:37Z" name: sequence-graph-6d218-57587c6648-hpj45.18be7e3271e26fe1 namespace: kserve-ci-e2e-test resourceVersion: "22102" uid: ecd963da-bc29-45b6-909a-41b1936509cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6d218} kind: Pod name: sequence-graph-6d218-57587c6648-hpj45 namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: 8d9d8706-f755-42c1-9fb7-8c31e211345d kind: Event lastTimestamp: "2026-07-02T14:07:37Z" message: Started container sequence-graph-6d218 metadata: creationTimestamp: "2026-07-02T14:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:37Z" name: sequence-graph-6d218-57587c6648-hpj45.18be7e3272535299 namespace: kserve-ci-e2e-test resourceVersion: "22103" uid: 2fb1f2b7-8156-4f08-8fcc-10f59ae45b3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6d218} kind: Pod name: sequence-graph-6d218-57587c6648-hpj45 namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: 8d9d8706-f755-42c1-9fb7-8c31e211345d kind: Event lastTimestamp: "2026-07-02T14:15:51Z" message: Stopping container sequence-graph-6d218 metadata: creationTimestamp: "2026-07-02T14:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:51Z" name: sequence-graph-6d218-57587c6648-hpj45.18be7ea57a9e920e namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: c980020c-4880-4f18-b170-0d3ff622a2fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:15:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-6d218} kind: Pod name: sequence-graph-6d218-57587c6648-hpj45 namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: 8d9d8706-f755-42c1-9fb7-8c31e211345d kind: Event lastTimestamp: "2026-07-02T14:16:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:15:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:19Z" name: sequence-graph-6d218-57587c6648-hpj45.18be7ea623316f33 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: 3cb69caa-1d3b-4ff4-bf60-395344f64d2e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-6d218-57587c6648 namespace: kserve-ci-e2e-test resourceVersion: "22072" uid: df547716-e693-4efe-8a69-10f9553e9f82 kind: Event lastTimestamp: "2026-07-02T14:07:37Z" message: 'Created pod: sequence-graph-6d218-57587c6648-hpj45' metadata: creationTimestamp: "2026-07-02T14:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:37Z" name: sequence-graph-6d218-57587c6648.18be7e325362b195 namespace: kserve-ci-e2e-test resourceVersion: "22078" uid: b925ac40-7237-44f4-abee-e81045a278d2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-6d218 namespace: kserve-ci-e2e-test resourceVersion: "22069" uid: efffc566-1d73-4468-9bcc-4ebc28e3efea kind: Event lastTimestamp: "2026-07-02T14:07:37Z" message: Scaled up replica set sequence-graph-6d218-57587c6648 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:37Z" name: sequence-graph-6d218.18be7e325241f846 namespace: kserve-ci-e2e-test resourceVersion: "22074" uid: a575d7f9-ddc4-4e62-812a-aa7d3903af52 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-6d218 namespace: kserve-ci-e2e-test resourceVersion: "22089" uid: 72084715-7791-4c88-8c4e-26e39f8df7ad kind: Event lastTimestamp: "2026-07-02T14:07:37Z" message: InferenceGraph [sequence-graph-6d218] is Ready metadata: creationTimestamp: "2026-07-02T14:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:07:37Z" name: sequence-graph-6d218.18be7e3256717ca4 namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: bd4e63c8-58c0-4707-994a-83b1b1845c02 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-6d218 namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: 48b872a6-b044-49cd-b1ba-a9029e157f0f kind: Event lastTimestamp: "2026-07-02T14:07:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:52Z" name: sequence-graph-6d218.18be7e35d0f4f1c4 namespace: kserve-ci-e2e-test resourceVersion: "22205" uid: 071b47e0-a099-44c3-8694-725b7597054f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-6d218 namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: 48b872a6-b044-49cd-b1ba-a9029e157f0f kind: Event lastTimestamp: "2026-07-02T14:07:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:52Z" name: sequence-graph-6d218.18be7e35d15bc828 namespace: kserve-ci-e2e-test resourceVersion: "22207" uid: 74604d1c-b56c-4cf6-95d1-16c5007641ab reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:57:11.904572Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-88827-56d8c9dfbf-5ll78 namespace: kserve-ci-e2e-test resourceVersion: "16832" uid: a74f0795-f1b7-4926-9b67-d7c3396b3f00 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-88827-56d8c9dfbf-5ll78 to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T13:57:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:57:11Z" name: sequence-graph-88827-56d8c9dfbf-5ll78.18be7da0c2f83541 namespace: kserve-ci-e2e-test resourceVersion: "16837" uid: b6de6e3a-2ab4-4851-b3f1-6222aaff6fa3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:12Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-88827-56d8c9dfbf-5ll78 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: a74f0795-f1b7-4926-9b67-d7c3396b3f00 kind: Event lastTimestamp: "2026-07-02T13:57:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-88827-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:12Z" name: sequence-graph-88827-56d8c9dfbf-5ll78.18be7da0cefbf950 namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 7872941a-be38-47b0-b698-bd95d2cd2ccc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:12Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-88827-56d8c9dfbf-5ll78 namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: a74f0795-f1b7-4926-9b67-d7c3396b3f00 kind: Event lastTimestamp: "2026-07-02T13:57:12Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:57:12Z" name: sequence-graph-88827-56d8c9dfbf-5ll78.18be7da100167d2c namespace: kserve-ci-e2e-test resourceVersion: "16861" uid: 0838bf44-9a11-4f21-b3ae-d8a4ce89bb9a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-88827} kind: Pod name: sequence-graph-88827-56d8c9dfbf-5ll78 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: a74f0795-f1b7-4926-9b67-d7c3396b3f00 kind: Event lastTimestamp: "2026-07-02T13:57:12Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" already present on machine metadata: creationTimestamp: "2026-07-02T13:57:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:12Z" name: sequence-graph-88827-56d8c9dfbf-5ll78.18be7da1014d3068 namespace: kserve-ci-e2e-test resourceVersion: "16863" uid: 96acfb96-e33e-48ac-b6f7-1ec0c57178d5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-88827} kind: Pod name: sequence-graph-88827-56d8c9dfbf-5ll78 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: a74f0795-f1b7-4926-9b67-d7c3396b3f00 kind: Event lastTimestamp: "2026-07-02T13:57:13Z" message: 'Created container: sequence-graph-88827' metadata: creationTimestamp: "2026-07-02T13:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:13Z" name: sequence-graph-88827-56d8c9dfbf-5ll78.18be7da10534db2b namespace: kserve-ci-e2e-test resourceVersion: "16864" uid: ab51c42d-5674-48f6-ab4f-d2abc313b41f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-88827} kind: Pod name: sequence-graph-88827-56d8c9dfbf-5ll78 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: a74f0795-f1b7-4926-9b67-d7c3396b3f00 kind: Event lastTimestamp: "2026-07-02T13:57:13Z" message: Started container sequence-graph-88827 metadata: creationTimestamp: "2026-07-02T13:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:57:13Z" name: sequence-graph-88827-56d8c9dfbf-5ll78.18be7da1059cc999 namespace: kserve-ci-e2e-test resourceVersion: "16865" uid: 970ef991-b218-4867-b524-a93374c35596 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-88827} kind: Pod name: sequence-graph-88827-56d8c9dfbf-5ll78 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: a74f0795-f1b7-4926-9b67-d7c3396b3f00 kind: Event lastTimestamp: "2026-07-02T14:05:26Z" message: Stopping container sequence-graph-88827 metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:26Z" name: sequence-graph-88827-56d8c9dfbf-5ll78.18be7e13f1c1e201 namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: 2abf1ec4-a17c-486a-8125-abc781ddc58d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-88827} kind: Pod name: sequence-graph-88827-56d8c9dfbf-5ll78 namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: a74f0795-f1b7-4926-9b67-d7c3396b3f00 kind: Event lastTimestamp: "2026-07-02T14:05:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:54Z" name: sequence-graph-88827-56d8c9dfbf-5ll78.18be7e14aa524cad namespace: kserve-ci-e2e-test resourceVersion: "20991" uid: b3a76310-2510-4855-a13a-87da844a7c56 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-88827-56d8c9dfbf namespace: kserve-ci-e2e-test resourceVersion: "16829" uid: cb81d8fc-04a2-4a50-9b3f-7aee08a5bb20 kind: Event lastTimestamp: "2026-07-02T13:57:11Z" message: 'Created pod: sequence-graph-88827-56d8c9dfbf-5ll78' metadata: creationTimestamp: "2026-07-02T13:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:11Z" name: sequence-graph-88827-56d8c9dfbf.18be7da0c27f0d89 namespace: kserve-ci-e2e-test resourceVersion: "16834" uid: 1a2512c9-bf95-41b2-b7d2-185f49a69c2f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-88827 namespace: kserve-ci-e2e-test resourceVersion: "16827" uid: 5bb3f690-2a83-407b-9ce6-91729f6682a7 kind: Event lastTimestamp: "2026-07-02T13:57:11Z" message: Scaled up replica set sequence-graph-88827-56d8c9dfbf from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:11Z" name: sequence-graph-88827.18be7da0c0fecb0b namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: c027c444-7a53-41a1-a813-0d2a93e2bec6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-88827 namespace: kserve-ci-e2e-test resourceVersion: "16846" uid: 5a3b9ddc-8b02-45fb-beda-411ccd8fc456 kind: Event lastTimestamp: "2026-07-02T13:57:11Z" message: InferenceGraph [sequence-graph-88827] is Ready metadata: creationTimestamp: "2026-07-02T13:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:57:11Z" name: sequence-graph-88827.18be7da0c863e5a3 namespace: kserve-ci-e2e-test resourceVersion: "16848" uid: 74d050f2-a9f0-4fa0-b184-b51777bedfaf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-88827 namespace: kserve-ci-e2e-test resourceVersion: "16828" uid: cb1437d4-eb75-473e-bf35-712ce0d5962a kind: Event lastTimestamp: "2026-07-02T13:57:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:26Z" name: sequence-graph-88827.18be7da43f8628e7 namespace: kserve-ci-e2e-test resourceVersion: "16963" uid: a787e262-e5bd-413a-ae1f-413679f5fb8a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-88827 namespace: kserve-ci-e2e-test resourceVersion: "16828" uid: cb1437d4-eb75-473e-bf35-712ce0d5962a kind: Event lastTimestamp: "2026-07-02T13:57:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:26Z" name: sequence-graph-88827.18be7da43fff85cb namespace: kserve-ci-e2e-test resourceVersion: "16965" uid: 03c57bb2-6479-409d-ad5a-fa1c90a96c4e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-88827 namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: cb1437d4-eb75-473e-bf35-712ce0d5962a kind: Event lastTimestamp: "2026-07-02T13:57:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:41Z" name: sequence-graph-88827.18be7da7bf765f7d namespace: kserve-ci-e2e-test resourceVersion: "17076" uid: ebbf6210-85f2-450a-9664-1b2488e4d638 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:57:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-88827 namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: cb1437d4-eb75-473e-bf35-712ce0d5962a kind: Event lastTimestamp: "2026-07-02T13:57:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:41Z" name: sequence-graph-88827.18be7da7bfceb7f2 namespace: kserve-ci-e2e-test resourceVersion: "17077" uid: 97eca022-8140-4eab-bcc7-2fd5a8fa654f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:06:26.799426Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c559a-57c5445779-r4zfm namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: aa15671a-0497-4910-81be-a7876a3f9e03 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-c559a-57c5445779-r4zfm to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:06:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:06:26Z" name: sequence-graph-c559a-57c5445779-r4zfm.18be7e21f547be6f namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 8ad7bdb0-2de4-4877-aae8-aa995d8cb3eb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:27Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c559a-57c5445779-r4zfm namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: aa15671a-0497-4910-81be-a7876a3f9e03 kind: Event lastTimestamp: "2026-07-02T14:06:27Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:06:27Z" name: sequence-graph-c559a-57c5445779-r4zfm.18be7e220f70cc55 namespace: kserve-ci-e2e-test resourceVersion: "21302" uid: b17c34ba-b3d5-4940-83bb-5fc0d83bc632 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c559a} kind: Pod name: sequence-graph-c559a-57c5445779-r4zfm namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: aa15671a-0497-4910-81be-a7876a3f9e03 kind: Event lastTimestamp: "2026-07-02T14:06:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" already present on machine metadata: creationTimestamp: "2026-07-02T14:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:27Z" name: sequence-graph-c559a-57c5445779-r4zfm.18be7e2210e2e295 namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: b02b1910-bd05-4c81-baf9-a17e48ab4604 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c559a} kind: Pod name: sequence-graph-c559a-57c5445779-r4zfm namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: aa15671a-0497-4910-81be-a7876a3f9e03 kind: Event lastTimestamp: "2026-07-02T14:06:27Z" message: 'Created container: sequence-graph-c559a' metadata: creationTimestamp: "2026-07-02T14:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:27Z" name: sequence-graph-c559a-57c5445779-r4zfm.18be7e22143c3579 namespace: kserve-ci-e2e-test resourceVersion: "21305" uid: d562bf5b-386a-4ea3-9a07-25cb1d3f6883 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c559a} kind: Pod name: sequence-graph-c559a-57c5445779-r4zfm namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: aa15671a-0497-4910-81be-a7876a3f9e03 kind: Event lastTimestamp: "2026-07-02T14:06:27Z" message: Started container sequence-graph-c559a metadata: creationTimestamp: "2026-07-02T14:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:27Z" name: sequence-graph-c559a-57c5445779-r4zfm.18be7e2214b76996 namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: 49e6b9ba-a9b0-424f-8946-6abb29359257 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c559a} kind: Pod name: sequence-graph-c559a-57c5445779-r4zfm namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: aa15671a-0497-4910-81be-a7876a3f9e03 kind: Event lastTimestamp: "2026-07-02T14:06:36Z" message: Stopping container sequence-graph-c559a metadata: creationTimestamp: "2026-07-02T14:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:36Z" name: sequence-graph-c559a-57c5445779-r4zfm.18be7e244e6d1434 namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: 4b3f72e2-fbf7-4704-ba21-cef8c41529cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c559a} kind: Pod name: sequence-graph-c559a-57c5445779-r4zfm namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: aa15671a-0497-4910-81be-a7876a3f9e03 kind: Event lastTimestamp: "2026-07-02T14:07:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:04Z" name: sequence-graph-c559a-57c5445779-r4zfm.18be7e24d7ded08b namespace: kserve-ci-e2e-test resourceVersion: "21796" uid: a6e9c785-fe3b-4cc2-807a-9a27554c7d8b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-c559a-57c5445779 namespace: kserve-ci-e2e-test resourceVersion: "21271" uid: 81638945-c178-4b4c-a5b1-537b67ce3430 kind: Event lastTimestamp: "2026-07-02T14:06:26Z" message: 'Created pod: sequence-graph-c559a-57c5445779-r4zfm' metadata: creationTimestamp: "2026-07-02T14:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:26Z" name: sequence-graph-c559a-57c5445779.18be7e21f4da063d namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: cef72752-c325-4fa9-b16e-9c0592a6fdbc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-c559a namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: 25e324d2-c936-4bec-bd42-9cd6e61cc553 kind: Event lastTimestamp: "2026-07-02T14:06:26Z" message: Scaled up replica set sequence-graph-c559a-57c5445779 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:26Z" name: sequence-graph-c559a.18be7e21f3f8b318 namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: d3316c3a-6d70-4361-ae6f-adbbbd37647e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c559a namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: ab3f300f-85ff-4135-8021-a241be9bb7d8 kind: Event lastTimestamp: "2026-07-02T14:06:26Z" message: InferenceGraph [sequence-graph-c559a] is Ready metadata: creationTimestamp: "2026-07-02T14:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:26Z" name: sequence-graph-c559a.18be7e21f9bf4fef namespace: kserve-ci-e2e-test resourceVersion: "21290" uid: 50726331-b3c9-429b-99aa-94a494e6c377 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:15:55.972574Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-09121-864f98c7fd-rj8ns namespace: kserve-ci-e2e-test resourceVersion: "26153" uid: 9147221c-2bcd-496d-a48f-e8cd49036e4d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-09121-864f98c7fd-rj8ns to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:15:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:15:55Z" name: splitter-graph-09121-864f98c7fd-rj8ns.18be7ea67aa4c765 namespace: kserve-ci-e2e-test resourceVersion: "26158" uid: 7ae84acf-5e8a-4b92-92c4-00fde4f28304 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:56Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-09121-864f98c7fd-rj8ns namespace: kserve-ci-e2e-test resourceVersion: "26164" uid: 9147221c-2bcd-496d-a48f-e8cd49036e4d kind: Event lastTimestamp: "2026-07-02T14:15:56Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:15:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:15:56Z" name: splitter-graph-09121-864f98c7fd-rj8ns.18be7ea6942ca5d1 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 9dc44618-06c5-44ad-b6e5-770cd5432c97 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-09121} kind: Pod name: splitter-graph-09121-864f98c7fd-rj8ns namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: 9147221c-2bcd-496d-a48f-e8cd49036e4d kind: Event lastTimestamp: "2026-07-02T14:15:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" already present on machine metadata: creationTimestamp: "2026-07-02T14:15:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:56Z" name: splitter-graph-09121-864f98c7fd-rj8ns.18be7ea695aef0e7 namespace: kserve-ci-e2e-test resourceVersion: "26175" uid: 9cf62f09-ec9d-47db-9f65-4a594c0a9083 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-09121} kind: Pod name: splitter-graph-09121-864f98c7fd-rj8ns namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: 9147221c-2bcd-496d-a48f-e8cd49036e4d kind: Event lastTimestamp: "2026-07-02T14:15:56Z" message: 'Created container: splitter-graph-09121' metadata: creationTimestamp: "2026-07-02T14:15:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:56Z" name: splitter-graph-09121-864f98c7fd-rj8ns.18be7ea698a15577 namespace: kserve-ci-e2e-test resourceVersion: "26176" uid: 9f0fb5a6-ed7a-484a-8062-ac47d11eaf56 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-09121} kind: Pod name: splitter-graph-09121-864f98c7fd-rj8ns namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: 9147221c-2bcd-496d-a48f-e8cd49036e4d kind: Event lastTimestamp: "2026-07-02T14:15:56Z" message: Started container splitter-graph-09121 metadata: creationTimestamp: "2026-07-02T14:15:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:56Z" name: splitter-graph-09121-864f98c7fd-rj8ns.18be7ea699142aa6 namespace: kserve-ci-e2e-test resourceVersion: "26177" uid: 7f336b95-c0a2-4e64-b054-88add31a666c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-09121} kind: Pod name: splitter-graph-09121-864f98c7fd-rj8ns namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: 9147221c-2bcd-496d-a48f-e8cd49036e4d kind: Event lastTimestamp: "2026-07-02T14:16:05Z" message: Stopping container splitter-graph-09121 metadata: creationTimestamp: "2026-07-02T14:16:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:05Z" name: splitter-graph-09121-864f98c7fd-rj8ns.18be7ea8d001402f namespace: kserve-ci-e2e-test resourceVersion: "26272" uid: db4de3fb-945a-4bda-a953-f5e95a901c65 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:16:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-09121} kind: Pod name: splitter-graph-09121-864f98c7fd-rj8ns namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: 9147221c-2bcd-496d-a48f-e8cd49036e4d kind: Event lastTimestamp: "2026-07-02T14:16:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:16:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:32Z" name: splitter-graph-09121-864f98c7fd-rj8ns.18be7ea935a6c6b3 namespace: kserve-ci-e2e-test resourceVersion: "26622" uid: 885364a1-ae32-4388-b36c-300df68f457f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-09121-864f98c7fd namespace: kserve-ci-e2e-test resourceVersion: "26149" uid: d076f1ad-4da2-4361-95e1-dc1121edcd99 kind: Event lastTimestamp: "2026-07-02T14:15:55Z" message: 'Created pod: splitter-graph-09121-864f98c7fd-rj8ns' metadata: creationTimestamp: "2026-07-02T14:15:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:55Z" name: splitter-graph-09121-864f98c7fd.18be7ea67a51c2cd namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: efb501e9-c3ec-4387-a9f1-ae5382ea4678 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-09121 namespace: kserve-ci-e2e-test resourceVersion: "26147" uid: 70f305dd-6dde-4cfc-a6f2-1f9d469db215 kind: Event lastTimestamp: "2026-07-02T14:15:55Z" message: Scaled up replica set splitter-graph-09121-864f98c7fd from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:15:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:55Z" name: splitter-graph-09121.18be7ea678c5d344 namespace: kserve-ci-e2e-test resourceVersion: "26151" uid: 460488d2-cca7-416d-abde-05bbe10232fe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-09121 namespace: kserve-ci-e2e-test resourceVersion: "26166" uid: 49b46eb0-a3e9-4541-8ebb-c23fa0ef2d10 kind: Event lastTimestamp: "2026-07-02T14:15:56Z" message: InferenceGraph [splitter-graph-09121] is Ready metadata: creationTimestamp: "2026-07-02T14:15:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:15:56Z" name: splitter-graph-09121.18be7ea67f5a8452 namespace: kserve-ci-e2e-test resourceVersion: "26168" uid: ed2ffe87-1fa0-4624-b5c2-e38d80ca906b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:17:06.207234Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-78590-85df59669f-6j9xq namespace: kserve-ci-e2e-test resourceVersion: "26974" uid: 6acd2f91-4605-4cfb-87a4-a3828dd4d960 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-78590-85df59669f-6j9xq to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:17:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:17:06Z" name: splitter-graph-78590-85df59669f-6j9xq.18be7eb6d4f489e4 namespace: kserve-ci-e2e-test resourceVersion: "26979" uid: b9c6d07d-ff00-441e-bec2-9f9dd7d79910 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:17:06Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-78590-85df59669f-6j9xq namespace: kserve-ci-e2e-test resourceVersion: "26975" uid: 6acd2f91-4605-4cfb-87a4-a3828dd4d960 kind: Event lastTimestamp: "2026-07-02T14:17:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-78590-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:17:06Z" name: splitter-graph-78590-85df59669f-6j9xq.18be7eb6e0b45765 namespace: kserve-ci-e2e-test resourceVersion: "26992" uid: bc8b046e-96f5-4992-ba91-214e177e8bc2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:17:07Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-78590-85df59669f-6j9xq namespace: kserve-ci-e2e-test resourceVersion: "26983" uid: 6acd2f91-4605-4cfb-87a4-a3828dd4d960 kind: Event lastTimestamp: "2026-07-02T14:17:07Z" message: Add eth0 [10.134.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:17:07Z" name: splitter-graph-78590-85df59669f-6j9xq.18be7eb7126f8951 namespace: kserve-ci-e2e-test resourceVersion: "27006" uid: 6d5942c0-2db5-45df-a1ef-25d0e2ac65a1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:17:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78590} kind: Pod name: splitter-graph-78590-85df59669f-6j9xq namespace: kserve-ci-e2e-test resourceVersion: "26975" uid: 6acd2f91-4605-4cfb-87a4-a3828dd4d960 kind: Event lastTimestamp: "2026-07-02T14:17:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" already present on machine metadata: creationTimestamp: "2026-07-02T14:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:17:07Z" name: splitter-graph-78590-85df59669f-6j9xq.18be7eb713dcfdba namespace: kserve-ci-e2e-test resourceVersion: "27008" uid: da56c5e5-978a-4724-8ba5-9e0e24d2fb7c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:17:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78590} kind: Pod name: splitter-graph-78590-85df59669f-6j9xq namespace: kserve-ci-e2e-test resourceVersion: "26975" uid: 6acd2f91-4605-4cfb-87a4-a3828dd4d960 kind: Event lastTimestamp: "2026-07-02T14:17:07Z" message: 'Created container: splitter-graph-78590' metadata: creationTimestamp: "2026-07-02T14:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:17:07Z" name: splitter-graph-78590-85df59669f-6j9xq.18be7eb716dc7be2 namespace: kserve-ci-e2e-test resourceVersion: "27010" uid: 37ff8a54-c1ac-44ed-a8a7-cf30809d859b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:17:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78590} kind: Pod name: splitter-graph-78590-85df59669f-6j9xq namespace: kserve-ci-e2e-test resourceVersion: "26975" uid: 6acd2f91-4605-4cfb-87a4-a3828dd4d960 kind: Event lastTimestamp: "2026-07-02T14:17:07Z" message: Started container splitter-graph-78590 metadata: creationTimestamp: "2026-07-02T14:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:17:07Z" name: splitter-graph-78590-85df59669f-6j9xq.18be7eb717a39547 namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: d5bfb3a2-e9af-4a96-8831-9ca2fe524d3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78590} kind: Pod name: splitter-graph-78590-85df59669f-6j9xq namespace: kserve-ci-e2e-test resourceVersion: "26975" uid: 6acd2f91-4605-4cfb-87a4-a3828dd4d960 kind: Event lastTimestamp: "2026-07-02T14:25:20Z" message: Stopping container splitter-graph-78590 metadata: creationTimestamp: "2026-07-02T14:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:20Z" name: splitter-graph-78590-85df59669f-6j9xq.18be7f29fecae907 namespace: kserve-ci-e2e-test resourceVersion: "30428" uid: 0be6248f-357b-4c82-9b3f-9cefd96e6165 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:25:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-78590} kind: Pod name: splitter-graph-78590-85df59669f-6j9xq namespace: kserve-ci-e2e-test resourceVersion: "26975" uid: 6acd2f91-4605-4cfb-87a4-a3828dd4d960 kind: Event lastTimestamp: "2026-07-02T14:25:49Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:25:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:49Z" name: splitter-graph-78590-85df59669f-6j9xq.18be7f2abe6bd68e namespace: kserve-ci-e2e-test resourceVersion: "30669" uid: a9e70219-cb15-4190-bea8-740d791fdefa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:17:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-78590-85df59669f namespace: kserve-ci-e2e-test resourceVersion: "26971" uid: ba496cbd-9ba4-409e-ab28-c902bea7dc9b kind: Event lastTimestamp: "2026-07-02T14:17:06Z" message: 'Created pod: splitter-graph-78590-85df59669f-6j9xq' metadata: creationTimestamp: "2026-07-02T14:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:06Z" name: splitter-graph-78590-85df59669f.18be7eb6d47d2ff9 namespace: kserve-ci-e2e-test resourceVersion: "26977" uid: 96a4f329-6410-4bd5-b0f7-8fdb878241b7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:17:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-78590 namespace: kserve-ci-e2e-test resourceVersion: "26968" uid: e3336085-edad-454f-a26f-d9689f16ba10 kind: Event lastTimestamp: "2026-07-02T14:17:06Z" message: Scaled up replica set splitter-graph-78590-85df59669f from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:06Z" name: splitter-graph-78590.18be7eb6d3b5e195 namespace: kserve-ci-e2e-test resourceVersion: "26972" uid: ac0a6099-6783-41d9-8172-81d0ab8563d3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:17:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-78590 namespace: kserve-ci-e2e-test resourceVersion: "26985" uid: 4f312721-10f3-4961-95e9-f8091b242ed7 kind: Event lastTimestamp: "2026-07-02T14:17:06Z" message: InferenceGraph [splitter-graph-78590] is Ready metadata: creationTimestamp: "2026-07-02T14:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:17:06Z" name: splitter-graph-78590.18be7eb6d9844f7f namespace: kserve-ci-e2e-test resourceVersion: "26987" uid: c9479d31-2222-4a2c-8536-92211b35495b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:17:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-78590 namespace: kserve-ci-e2e-test resourceVersion: "26969" uid: ef3e8b04-2733-422c-8a64-53b958344c4c kind: Event lastTimestamp: "2026-07-02T14:17:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:17:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:21Z" name: splitter-graph-78590.18be7eba5248c3fa namespace: kserve-ci-e2e-test resourceVersion: "27111" uid: 3595f3b6-d0e8-4f9c-8d26-bec550668d37 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:17:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-78590 namespace: kserve-ci-e2e-test resourceVersion: "26969" uid: ef3e8b04-2733-422c-8a64-53b958344c4c kind: Event lastTimestamp: "2026-07-02T14:17:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:17:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:21Z" name: splitter-graph-78590.18be7eba52a701a6 namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: b2635ca1-0920-46be-816e-b5fc2d2c93c3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:14:55.874858Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm namespace: kserve-ci-e2e-test resourceVersion: "25131" uid: 4ae79875-c7f8-4433-ba70-8d2d76b10627 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-09121-predictor-66bcd6b46-k2xnm to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:14:55Z" name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm.18be7e987c8d5276 namespace: kserve-ci-e2e-test resourceVersion: "25142" uid: ed868b6c-00ed-4827-9594-36001c983fe2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm namespace: kserve-ci-e2e-test resourceVersion: "25154" uid: 4ae79875-c7f8-4433-ba70-8d2d76b10627 kind: Event lastTimestamp: "2026-07-02T14:14:56Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:14:56Z" name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm.18be7e989633a120 namespace: kserve-ci-e2e-test resourceVersion: "25201" uid: 38fd2f7a-77df-4dcc-a9e2-dd101203a335 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 4ae79875-c7f8-4433-ba70-8d2d76b10627 kind: Event lastTimestamp: "2026-07-02T14:14:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:56Z" name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm.18be7e9897b50855 namespace: kserve-ci-e2e-test resourceVersion: "25205" uid: b5ea4a17-45c2-47ce-bae2-80e0905be349 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 4ae79875-c7f8-4433-ba70-8d2d76b10627 kind: Event lastTimestamp: "2026-07-02T14:14:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:56Z" name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm.18be7e989aaff785 namespace: kserve-ci-e2e-test resourceVersion: "25207" uid: 00e9025b-e678-4ede-8032-50ae1a84e789 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 4ae79875-c7f8-4433-ba70-8d2d76b10627 kind: Event lastTimestamp: "2026-07-02T14:14:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:56Z" name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm.18be7e989b1e98f1 namespace: kserve-ci-e2e-test resourceVersion: "25209" uid: 49fa8fd3-6236-4247-af9d-9e1fc70fab86 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 4ae79875-c7f8-4433-ba70-8d2d76b10627 kind: Event lastTimestamp: "2026-07-02T14:14:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:56Z" name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm.18be7e989b273f76 namespace: kserve-ci-e2e-test resourceVersion: "25210" uid: 45643cde-1815-48e7-8150-46f272015fcf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 4ae79875-c7f8-4433-ba70-8d2d76b10627 kind: Event lastTimestamp: "2026-07-02T14:14:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:56Z" name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm.18be7e989dfef0f7 namespace: kserve-ci-e2e-test resourceVersion: "25212" uid: 4a514da2-5c84-4e8f-a5c5-da52d8773e7a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 4ae79875-c7f8-4433-ba70-8d2d76b10627 kind: Event lastTimestamp: "2026-07-02T14:14:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:14:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:56Z" name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm.18be7e989e6ee288 namespace: kserve-ci-e2e-test resourceVersion: "25213" uid: 0739c588-4e69-4142-bb05-bc5ed3240239 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:14:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 4ae79875-c7f8-4433-ba70-8d2d76b10627 kind: Event lastTimestamp: "2026-07-02T14:15:34Z" message: 'Readiness probe failed: dial tcp 10.134.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:34Z" name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm.18be7e99143631c5 namespace: kserve-ci-e2e-test resourceVersion: "25795" uid: 707c8d72-448a-4d5f-95b5-f2a00617f389 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 4ae79875-c7f8-4433-ba70-8d2d76b10627 kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm.18be7ea8d853227a namespace: kserve-ci-e2e-test resourceVersion: "26307" uid: 57bed70f-980c-4252-ba93-a1774e7c081f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: 4ae79875-c7f8-4433-ba70-8d2d76b10627 kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-09121-predictor-66bcd6b46-k2xnm.18be7ea8d854591d namespace: kserve-ci-e2e-test resourceVersion: "26311" uid: 969c5fb3-fbc0-4c7a-a11f-c379b081ebfa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-09121-predictor-66bcd6b46 namespace: kserve-ci-e2e-test resourceVersion: "25116" uid: 01ba4742-e136-471e-b958-e4599cc77236 kind: Event lastTimestamp: "2026-07-02T14:14:55Z" message: 'Created pod: success-200-isvc-09121-predictor-66bcd6b46-k2xnm' metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:14:55Z" name: success-200-isvc-09121-predictor-66bcd6b46.18be7e987c07995b namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: 4bf6c72c-b2c1-4b99-aa81-720bb77b48d5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-09121-predictor namespace: kserve-ci-e2e-test resourceVersion: "25114" uid: 49d2dc39-1b01-47e2-ba63-6539c37424d9 kind: Event lastTimestamp: "2026-07-02T14:14:55Z" message: Scaled up replica set success-200-isvc-09121-predictor-66bcd6b46 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:14:55Z" name: success-200-isvc-09121-predictor.18be7e98781f0e69 namespace: kserve-ci-e2e-test resourceVersion: "25117" uid: 78691d2f-c9e1-4154-aea6-114697c20371 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:15:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-09121-predictor namespace: kserve-ci-e2e-test resourceVersion: "25115" uid: 8990d3e3-480c-4bf3-a422-f4a5105d17f7 kind: Event lastTimestamp: "2026-07-02T14:15:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:25Z" name: success-200-isvc-09121-predictor.18be7e9bf6a67937 namespace: kserve-ci-e2e-test resourceVersion: "25719" uid: 17844cc4-c466-4071-97f9-4c0fd58f461f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:15:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-09121-predictor namespace: kserve-ci-e2e-test resourceVersion: "25115" uid: 8990d3e3-480c-4bf3-a422-f4a5105d17f7 kind: Event lastTimestamp: "2026-07-02T14:15:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:25Z" name: success-200-isvc-09121-predictor.18be7e9bf71b7f38 namespace: kserve-ci-e2e-test resourceVersion: "25720" uid: a868cfd2-e5c7-42b0-936a-149f3f37decd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:15:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-09121-predictor namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 8990d3e3-480c-4bf3-a422-f4a5105d17f7 kind: Event lastTimestamp: "2026-07-02T14:15:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:55Z" name: success-200-isvc-09121-predictor.18be7ea2f4a95475 namespace: kserve-ci-e2e-test resourceVersion: "26141" uid: 55fd4088-bd62-43bc-a8dd-63cc53b64fe0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:15:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-09121-predictor namespace: kserve-ci-e2e-test resourceVersion: "25341" uid: 8990d3e3-480c-4bf3-a422-f4a5105d17f7 kind: Event lastTimestamp: "2026-07-02T14:15:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:55Z" name: success-200-isvc-09121-predictor.18be7ea2f51ab7d1 namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: ab81b51c-a477-467c-8a6b-5d381f576e1e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:14:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-09121 namespace: kserve-ci-e2e-test resourceVersion: "25106" uid: 669793f1-4498-4036-861a-c9453db05f5a kind: Event lastTimestamp: "2026-07-02T14:15:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:15:44Z" name: success-200-isvc-09121.18be7e987826530a namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: f5879114-dc35-4f02-b679-b42c593ac878 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-09121 namespace: kserve-ci-e2e-test resourceVersion: "25106" uid: 669793f1-4498-4036-861a-c9453db05f5a kind: Event lastTimestamp: "2026-07-02T14:14:55Z" message: 'Failed to update status for InferenceService "success-200-isvc-09121": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-09121": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:14:55Z" name: success-200-isvc-09121.18be7e9878b12bd0 namespace: kserve-ci-e2e-test resourceVersion: "25120" uid: bc9b24d3-468f-4248-ae7f-2b1905d10f4b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-09121 namespace: kserve-ci-e2e-test resourceVersion: "25106" uid: 669793f1-4498-4036-861a-c9453db05f5a kind: Event lastTimestamp: "2026-07-02T14:14:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-09121": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:14:55Z" name: success-200-isvc-09121.18be7e9878b17951 namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: 44d42df4-b392-4084-8bc0-a712ad88c809 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-09121 namespace: kserve-ci-e2e-test resourceVersion: "25907" uid: 669793f1-4498-4036-861a-c9453db05f5a kind: Event lastTimestamp: "2026-07-02T14:15:44Z" message: InferenceService [success-200-isvc-09121] is Ready metadata: creationTimestamp: "2026-07-02T14:15:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:15:44Z" name: success-200-isvc-09121.18be7ea3cfcd8872 namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: 7fa14568-c339-4172-b717-223a6d4c4576 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:53:41.453494Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14408" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-40523-predictor-54dbf486cb-2sn47 to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:53:41Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d6fc31b9e0b namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: 8f50aa89-828c-49ae-877a-5dc9d3f07ed1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: "2026-07-02T13:53:41Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:53:41Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d6fdcdd4d8f namespace: kserve-ci-e2e-test resourceVersion: "14494" uid: 2c30fead-d061-4ce1-8520-b58017fef735 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: "2026-07-02T13:53:41Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1695" metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:41Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d6fde51de9d namespace: kserve-ci-e2e-test resourceVersion: "14497" uid: 87b2de3e-d6eb-4ddc-b9bd-a612c0a9940d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: "2026-07-02T13:53:54Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1695" in 12.296s (12.296s including waiting). Image size: 1335687809 bytes.' metadata: creationTimestamp: "2026-07-02T13:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:54Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d72bb461ce2 namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: f0925cff-eb01-429e-b06c-5c5e4ba8dfc4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: "2026-07-02T13:53:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:54Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d72bebc007b namespace: kserve-ci-e2e-test resourceVersion: "14682" uid: c1de9fdb-efd6-48fd-942d-a36be16c8bce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: "2026-07-02T13:53:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:54Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d72bf1fe395 namespace: kserve-ci-e2e-test resourceVersion: "14683" uid: 9b66fd4f-9017-4575-aa0c-f1b7b336fbc3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: "2026-07-02T13:53:54Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T13:53:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:54Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d72bf2af407 namespace: kserve-ci-e2e-test resourceVersion: "14684" uid: 5ae22069-6069-4b33-9c53-73577a0d69d3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: "2026-07-02T13:53:56Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.257s (2.257s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:56Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d7345bbd8bb namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: 59a3b3f8-8e5e-44d4-bcbb-f8c871ff7e0a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: "2026-07-02T13:53:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:56Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d73488ef36d namespace: kserve-ci-e2e-test resourceVersion: "14713" uid: 7df8c088-12e1-4252-bd9b-5b56e1d3a6a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: "2026-07-02T13:53:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:53:56Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d7348f139eb namespace: kserve-ci-e2e-test resourceVersion: "14714" uid: b932144a-690f-49d1-84f2-65dc4d77d0e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: "2026-07-02T13:54:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:54:32Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d7357c56433 namespace: kserve-ci-e2e-test resourceVersion: "15050" uid: 554bff64-9776-4207-93b4-b74dc5897ed7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: "2026-07-02T13:55:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:15Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d85be867523 namespace: kserve-ci-e2e-test resourceVersion: "15515" uid: 6463dc9a-0607-400a-a70f-7cd2f1805046 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: "2026-07-02T13:55:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:15Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d85be87bd2a namespace: kserve-ci-e2e-test resourceVersion: "15518" uid: 4042038b-5e9c-4a05-9039-1d7b3b9e2462 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-40523-predictor-54dbf486cb-2sn47 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: 7e1331f3-f255-450a-be3f-18d5eb2a2de9 kind: Event lastTimestamp: "2026-07-02T13:55:17Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:17Z" name: success-200-isvc-40523-predictor-54dbf486cb-2sn47.18be7d8633fb467d namespace: kserve-ci-e2e-test resourceVersion: "15598" uid: 600b2cf1-d527-4ad0-b9c1-0ad6f0637a2f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-40523-predictor-54dbf486cb namespace: kserve-ci-e2e-test resourceVersion: "14397" uid: a90e978e-661c-492f-a1f8-b2166aa8aa07 kind: Event lastTimestamp: "2026-07-02T13:53:41Z" message: 'Created pod: success-200-isvc-40523-predictor-54dbf486cb-2sn47' metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:41Z" name: success-200-isvc-40523-predictor-54dbf486cb.18be7d6fc2c233de namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: eff9d24c-f750-4ab7-ae92-d0363900ea41 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-40523-predictor namespace: kserve-ci-e2e-test resourceVersion: "14396" uid: b27e5781-5bb2-40b6-8408-16af8909d4a8 kind: Event lastTimestamp: "2026-07-02T13:53:41Z" message: Scaled up replica set success-200-isvc-40523-predictor-54dbf486cb from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:53:41Z" name: success-200-isvc-40523-predictor.18be7d6fbc22793e namespace: kserve-ci-e2e-test resourceVersion: "14399" uid: cff52708-2caf-4329-9031-4686259292dd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-40523-predictor namespace: kserve-ci-e2e-test resourceVersion: "14405" uid: 28710e3e-d789-48c0-8809-db5ea7dcff01 kind: Event lastTimestamp: "2026-07-02T13:54:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:54:26Z" name: success-200-isvc-40523-predictor.18be7d7343115ea7 namespace: kserve-ci-e2e-test resourceVersion: "14990" uid: 03334050-08c7-41c7-800f-41395d222510 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:53:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-40523-predictor namespace: kserve-ci-e2e-test resourceVersion: "14405" uid: 28710e3e-d789-48c0-8809-db5ea7dcff01 kind: Event lastTimestamp: "2026-07-02T13:54:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:53:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:54:26Z" name: success-200-isvc-40523-predictor.18be7d7343d76fbc namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: 1d74c8f8-7e04-4474-a4ec-428996cf2d77 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:54:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-40523-predictor namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: 28710e3e-d789-48c0-8809-db5ea7dcff01 kind: Event lastTimestamp: "2026-07-02T13:55:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:11Z" name: success-200-isvc-40523-predictor.18be7d7dc106694f namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: 817e3480-e1a4-4a78-b2d9-3e045f0441c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:54:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-40523-predictor namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: 28710e3e-d789-48c0-8809-db5ea7dcff01 kind: Event lastTimestamp: "2026-07-02T13:55:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:54:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:11Z" name: success-200-isvc-40523-predictor.18be7d7dc171b47a namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: a40f6bbb-e0b4-4007-90bf-e0efd5149d1a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:53:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-40523 namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: c1ad683a-cd78-4422-b4cf-fc3c6271c51a kind: Event lastTimestamp: "2026-07-02T13:54:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:53:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:54:42Z" name: success-200-isvc-40523.18be7d6fc2264a7f namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: fe718fc7-84f4-4e30-ab96-5d4bfae3ecd2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:54:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-40523 namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: c1ad683a-cd78-4422-b4cf-fc3c6271c51a kind: Event lastTimestamp: "2026-07-02T13:54:42Z" message: InferenceService [success-200-isvc-40523] is Ready metadata: creationTimestamp: "2026-07-02T13:54:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:54:42Z" name: success-200-isvc-40523.18be7d7e1407f836 namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 658b30ee-ce70-460f-a814-1005696cf960 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:55:15.839530Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-68692-predictor-98ddcc858-6r58m namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: 0dd64166-d6c9-4373-ac9b-b5d692fd6f62 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-68692-predictor-98ddcc858-6r58m to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:55:15Z" name: success-200-isvc-68692-predictor-98ddcc858-6r58m.18be7d85bcf42e6a namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: f3fb683d-5ab0-4bf8-bff2-753be53220dd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-68692-predictor-98ddcc858-6r58m namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 0dd64166-d6c9-4373-ac9b-b5d692fd6f62 kind: Event lastTimestamp: "2026-07-02T13:55:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-68692-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:16Z" name: success-200-isvc-68692-predictor-98ddcc858-6r58m.18be7d85c874ecea namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: 51a53536-14c3-40ad-9aae-b460bd39a0a4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-68692-predictor-98ddcc858-6r58m namespace: kserve-ci-e2e-test resourceVersion: "15520" uid: 0dd64166-d6c9-4373-ac9b-b5d692fd6f62 kind: Event lastTimestamp: "2026-07-02T13:55:16Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:55:16Z" name: success-200-isvc-68692-predictor-98ddcc858-6r58m.18be7d85fa0a513a namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: 04d0eb17-ee99-46af-bd47-1249002ac237 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68692-predictor-98ddcc858-6r58m namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 0dd64166-d6c9-4373-ac9b-b5d692fd6f62 kind: Event lastTimestamp: "2026-07-02T13:55:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T13:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:16Z" name: success-200-isvc-68692-predictor-98ddcc858-6r58m.18be7d85fbb5136d namespace: kserve-ci-e2e-test resourceVersion: "15575" uid: 048db863-cd7a-4c39-b886-d228ce1f507f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68692-predictor-98ddcc858-6r58m namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 0dd64166-d6c9-4373-ac9b-b5d692fd6f62 kind: Event lastTimestamp: "2026-07-02T13:55:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:16Z" name: success-200-isvc-68692-predictor-98ddcc858-6r58m.18be7d85fe5043ab namespace: kserve-ci-e2e-test resourceVersion: "15576" uid: 0278b3c9-3176-4f9a-a478-3e4a4d7aef33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68692-predictor-98ddcc858-6r58m namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 0dd64166-d6c9-4373-ac9b-b5d692fd6f62 kind: Event lastTimestamp: "2026-07-02T13:55:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:16Z" name: success-200-isvc-68692-predictor-98ddcc858-6r58m.18be7d85fee0413c namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: c4524c81-4bf2-4e64-8c9f-63cbccec3b16 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68692-predictor-98ddcc858-6r58m namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 0dd64166-d6c9-4373-ac9b-b5d692fd6f62 kind: Event lastTimestamp: "2026-07-02T13:55:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:16Z" name: success-200-isvc-68692-predictor-98ddcc858-6r58m.18be7d85feea56e9 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 9a87df08-d244-448c-b2a9-1d53067b40e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68692-predictor-98ddcc858-6r58m namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 0dd64166-d6c9-4373-ac9b-b5d692fd6f62 kind: Event lastTimestamp: "2026-07-02T13:55:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:55:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:16Z" name: success-200-isvc-68692-predictor-98ddcc858-6r58m.18be7d8601cdae00 namespace: kserve-ci-e2e-test resourceVersion: "15582" uid: a4daea6a-27db-408b-9209-577b9f336ee1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68692-predictor-98ddcc858-6r58m namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 0dd64166-d6c9-4373-ac9b-b5d692fd6f62 kind: Event lastTimestamp: "2026-07-02T13:55:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:17Z" name: success-200-isvc-68692-predictor-98ddcc858-6r58m.18be7d86022fd45e namespace: kserve-ci-e2e-test resourceVersion: "15583" uid: ffb955c3-7116-422d-9384-2447b4a37d92 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68692-predictor-98ddcc858-6r58m namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 0dd64166-d6c9-4373-ac9b-b5d692fd6f62 kind: Event lastTimestamp: "2026-07-02T14:04:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:33Z" name: success-200-isvc-68692-predictor-98ddcc858-6r58m.18be7d860c7d010d namespace: kserve-ci-e2e-test resourceVersion: "20009" uid: d9958ea3-2560-48c1-9770-80092fa123a6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68692-predictor-98ddcc858-6r58m namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 0dd64166-d6c9-4373-ac9b-b5d692fd6f62 kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:30Z" name: success-200-isvc-68692-predictor-98ddcc858-6r58m.18be7e06e6fd456f namespace: kserve-ci-e2e-test resourceVersion: "19909" uid: 43ad91e3-bee2-4fc5-ab8f-0c9b64165a41 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68692-predictor-98ddcc858-6r58m namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 0dd64166-d6c9-4373-ac9b-b5d692fd6f62 kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:30Z" name: success-200-isvc-68692-predictor-98ddcc858-6r58m.18be7e06e6fe2521 namespace: kserve-ci-e2e-test resourceVersion: "19912" uid: 3ba0fe7e-b931-4793-a0fc-dd9e93a3168d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68692-predictor-98ddcc858-6r58m namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: 0dd64166-d6c9-4373-ac9b-b5d692fd6f62 kind: Event lastTimestamp: "2026-07-02T14:04:33Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:33Z" name: success-200-isvc-68692-predictor-98ddcc858-6r58m.18be7e0780e1e381 namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: ffc46b13-49e0-45dd-b727-716b89f90faa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-68692-predictor-98ddcc858 namespace: kserve-ci-e2e-test resourceVersion: "15477" uid: d284dc4d-92a1-4951-968e-1b75fe060d3c kind: Event lastTimestamp: "2026-07-02T13:55:15Z" message: 'Created pod: success-200-isvc-68692-predictor-98ddcc858-6r58m' metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:15Z" name: success-200-isvc-68692-predictor-98ddcc858.18be7d85bc26068a namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: 0534e6f6-8b91-42bb-a55c-c882b22da72f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-68692-predictor namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: 1f3a897d-9781-4444-ad7a-955f77ffaedb kind: Event lastTimestamp: "2026-07-02T13:55:15Z" message: Scaled up replica set success-200-isvc-68692-predictor-98ddcc858 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:15Z" name: success-200-isvc-68692-predictor.18be7d85b81f3b68 namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: 6d6be0ff-4922-4520-9c8d-950075a8d6ae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:55:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68692-predictor namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: 76bb47f5-21fe-4267-8c53-875750d0b496 kind: Event lastTimestamp: "2026-07-02T13:56:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:00Z" name: success-200-isvc-68692-predictor.18be7d8936a625b9 namespace: kserve-ci-e2e-test resourceVersion: "15985" uid: 77ce750e-9bdc-43cc-8fc5-e23cb3c32895 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:55:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68692-predictor namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: 76bb47f5-21fe-4267-8c53-875750d0b496 kind: Event lastTimestamp: "2026-07-02T13:56:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:00Z" name: success-200-isvc-68692-predictor.18be7d89370af488 namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: c7f92965-fa61-4f82-9420-d7709a28d479 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:56:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68692-predictor namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: 76bb47f5-21fe-4267-8c53-875750d0b496 kind: Event lastTimestamp: "2026-07-02T13:56:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:30Z" name: success-200-isvc-68692-predictor.18be7d93b393db55 namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: 2acba309-6346-4127-95f4-872518a077da reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:56:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68692-predictor namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: 76bb47f5-21fe-4267-8c53-875750d0b496 kind: Event lastTimestamp: "2026-07-02T13:56:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:30Z" name: success-200-isvc-68692-predictor.18be7d93b409203a namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: c263f32d-2fd5-41aa-b589-7a44944314a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68692 namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 81fc575f-e4a4-4fee-a09c-4ac512c2a482 kind: Event lastTimestamp: "2026-07-02T13:56:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:56:03Z" name: success-200-isvc-68692.18be7d85b828e1fc namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: e4901d67-db0f-45b5-97c9-7b194738050a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68692 namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 81fc575f-e4a4-4fee-a09c-4ac512c2a482 kind: Event lastTimestamp: "2026-07-02T13:55:15Z" message: 'Failed to update status for InferenceService "success-200-isvc-68692": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-68692": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:15Z" name: success-200-isvc-68692.18be7d85b8ad140c namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: 3d3b8385-7d82-407c-8d6c-545cdf48e080 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68692 namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 81fc575f-e4a4-4fee-a09c-4ac512c2a482 kind: Event lastTimestamp: "2026-07-02T13:55:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-68692": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:15Z" name: success-200-isvc-68692.18be7d85b8ad5b4a namespace: kserve-ci-e2e-test resourceVersion: "15482" uid: 8df6a373-2678-4482-a112-c9d3c0ecbbd9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68692 namespace: kserve-ci-e2e-test resourceVersion: "16045" uid: 81fc575f-e4a4-4fee-a09c-4ac512c2a482 kind: Event lastTimestamp: "2026-07-02T13:56:03Z" message: InferenceService [success-200-isvc-68692] is Ready metadata: creationTimestamp: "2026-07-02T13:56:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:56:03Z" name: success-200-isvc-68692.18be7d90c8a1af0e namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 86a3e9ce-1c1e-4ef3-bbf4-6a52f895b74b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:06:37.015266Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl namespace: kserve-ci-e2e-test resourceVersion: "21456" uid: 5fa21fbc-9387-4a9b-a614-cea780b794e3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:06:37Z" name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl.18be7e2456310f23 namespace: kserve-ci-e2e-test resourceVersion: "21464" uid: f875f33e-2eb9-41f7-9864-45287903ffea reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl namespace: kserve-ci-e2e-test resourceVersion: "21473" uid: 5fa21fbc-9387-4a9b-a614-cea780b794e3 kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:06:37Z" name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl.18be7e24700ed059 namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: 6bc787ac-2f30-4fd4-9856-f952ace5c23c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 5fa21fbc-9387-4a9b-a614-cea780b794e3 kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:37Z" name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl.18be7e2471630248 namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: 279222b1-6217-489f-b8d7-fed5a7ea3636 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 5fa21fbc-9387-4a9b-a614-cea780b794e3 kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:37Z" name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl.18be7e2473e1b311 namespace: kserve-ci-e2e-test resourceVersion: "21517" uid: dc0b6f97-23a8-471c-bb29-f1d5a5bfc08e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 5fa21fbc-9387-4a9b-a614-cea780b794e3 kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:37Z" name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl.18be7e24747f0af3 namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 4e93a53d-0c07-44b3-923a-a49c1f9b529e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 5fa21fbc-9387-4a9b-a614-cea780b794e3 kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:37Z" name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl.18be7e2474881bc9 namespace: kserve-ci-e2e-test resourceVersion: "21519" uid: 0c94a64c-6c83-4dfc-a0bd-717a77320a24 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 5fa21fbc-9387-4a9b-a614-cea780b794e3 kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:37Z" name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl.18be7e24772756b4 namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: 661ccf71-f822-4c92-85e6-06ee3ef57230 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 5fa21fbc-9387-4a9b-a614-cea780b794e3 kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:37Z" name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl.18be7e24779b30dc namespace: kserve-ci-e2e-test resourceVersion: "21523" uid: 9a4a653a-f46c-4ea5-befd-9733388937c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 5fa21fbc-9387-4a9b-a614-cea780b794e3 kind: Event lastTimestamp: "2026-07-02T14:07:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:07:15Z" name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl.18be7e24dab3b5c2 namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: 65c53e68-a293-4b11-a676-412164f9e1ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 5fa21fbc-9387-4a9b-a614-cea780b794e3 kind: Event lastTimestamp: "2026-07-02T14:15:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:51Z" name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl.18be7ea58373ed19 namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: a066ef74-c8b3-41eb-9b09-a11f7c75185e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl namespace: kserve-ci-e2e-test resourceVersion: "21460" uid: 5fa21fbc-9387-4a9b-a614-cea780b794e3 kind: Event lastTimestamp: "2026-07-02T14:15:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:51Z" name: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl.18be7ea58374857f namespace: kserve-ci-e2e-test resourceVersion: "26010" uid: 09d16543-8f52-4d3f-a574-fb6a4535495a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6d218-predictor-6fbfd44c8 namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: 1210ac73-3240-4bfb-bd44-5e925ab6f126 kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: 'Created pod: success-200-isvc-6d218-predictor-6fbfd44c8-7cpcl' metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:37Z" name: success-200-isvc-6d218-predictor-6fbfd44c8.18be7e2455a5e0ec namespace: kserve-ci-e2e-test resourceVersion: "21458" uid: 9449d960-678c-4a3f-b149-40ae67e0fe73 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6d218-predictor namespace: kserve-ci-e2e-test resourceVersion: "21435" uid: 1f256d80-732d-4b09-bf55-b4da2e0b6402 kind: Event lastTimestamp: "2026-07-02T14:06:36Z" message: Scaled up replica set success-200-isvc-6d218-predictor-6fbfd44c8 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:36Z" name: success-200-isvc-6d218-predictor.18be7e2453fe6970 namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: d72dcb19-cdc4-4ceb-942a-2caffa5d5fbf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:06:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6d218-predictor namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: 1f1ae88e-4e0f-45e1-9356-5228137964a7 kind: Event lastTimestamp: "2026-07-02T14:07:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:37Z" name: success-200-isvc-6d218-predictor.18be7e27d2598ad3 namespace: kserve-ci-e2e-test resourceVersion: "22070" uid: 206ee0f0-04e8-4fe8-9d50-613cda5e1ee3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:06:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6d218-predictor namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: 1f1ae88e-4e0f-45e1-9356-5228137964a7 kind: Event lastTimestamp: "2026-07-02T14:07:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:37Z" name: success-200-isvc-6d218-predictor.18be7e27d2c4a3ad namespace: kserve-ci-e2e-test resourceVersion: "22073" uid: 527739c7-562e-4fb9-99d3-9735944f5700 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6d218-predictor namespace: kserve-ci-e2e-test resourceVersion: "21695" uid: 1f1ae88e-4e0f-45e1-9356-5228137964a7 kind: Event lastTimestamp: "2026-07-02T14:07:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:07Z" name: success-200-isvc-6d218-predictor.18be7e2b52642512 namespace: kserve-ci-e2e-test resourceVersion: "21818" uid: 2a7e35ff-b831-4cf4-a1dc-b742c5e40ded reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6d218-predictor namespace: kserve-ci-e2e-test resourceVersion: "21695" uid: 1f1ae88e-4e0f-45e1-9356-5228137964a7 kind: Event lastTimestamp: "2026-07-02T14:07:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:07:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:07:07Z" name: success-200-isvc-6d218-predictor.18be7e2b52d54d35 namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: 518bbcad-059e-47f5-8550-9bf078a95adf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:06:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6d218 namespace: kserve-ci-e2e-test resourceVersion: "21416" uid: d39ef311-e4f0-4854-bb23-a875b44c71c9 kind: Event lastTimestamp: "2026-07-02T14:07:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:06:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:07:25Z" name: success-200-isvc-6d218.18be7e2453fdfa26 namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: 36d580db-aaec-44d2-93d8-65b039d2204e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:07:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6d218 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: d39ef311-e4f0-4854-bb23-a875b44c71c9 kind: Event lastTimestamp: "2026-07-02T14:07:25Z" message: InferenceService [success-200-isvc-6d218] is Ready metadata: creationTimestamp: "2026-07-02T14:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:07:25Z" name: success-200-isvc-6d218.18be7e2f97869dcf namespace: kserve-ci-e2e-test resourceVersion: "21971" uid: a5886608-ee23-4af9-8322-eb0cfed9b363 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:16:06.149190Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78590-predictor-599b7b7f46-8r44l namespace: kserve-ci-e2e-test resourceVersion: "26306" uid: 24a1816a-ea79-42c5-af42-a3f922f1526e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-78590-predictor-599b7b7f46-8r44l to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-78590-predictor-599b7b7f46-8r44l.18be7ea8d937a472 namespace: kserve-ci-e2e-test resourceVersion: "26313" uid: dd7d042c-813b-4c89-86d0-af9e79aa1ff3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-78590-predictor-599b7b7f46-8r44l namespace: kserve-ci-e2e-test resourceVersion: "26318" uid: 24a1816a-ea79-42c5-af42-a3f922f1526e kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-78590-predictor-599b7b7f46-8r44l.18be7ea8f31acdc6 namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: 300884f1-847a-4472-be95-29b48e3131cd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78590-predictor-599b7b7f46-8r44l namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 24a1816a-ea79-42c5-af42-a3f922f1526e kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-78590-predictor-599b7b7f46-8r44l.18be7ea8f47ef04c namespace: kserve-ci-e2e-test resourceVersion: "26373" uid: aae3bb33-7f19-493e-8799-de5597bf96a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78590-predictor-599b7b7f46-8r44l namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 24a1816a-ea79-42c5-af42-a3f922f1526e kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-78590-predictor-599b7b7f46-8r44l.18be7ea8f7450596 namespace: kserve-ci-e2e-test resourceVersion: "26375" uid: 32890e5e-2ce5-4d8a-84df-aa350ff54831 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78590-predictor-599b7b7f46-8r44l namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 24a1816a-ea79-42c5-af42-a3f922f1526e kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-78590-predictor-599b7b7f46-8r44l.18be7ea8f7b55909 namespace: kserve-ci-e2e-test resourceVersion: "26376" uid: b1e91281-c233-4284-9af5-8db005c75e71 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78590-predictor-599b7b7f46-8r44l namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 24a1816a-ea79-42c5-af42-a3f922f1526e kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-78590-predictor-599b7b7f46-8r44l.18be7ea8f7c159ea namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: ebc4e09f-f903-4197-b592-b3add10128df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78590-predictor-599b7b7f46-8r44l namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 24a1816a-ea79-42c5-af42-a3f922f1526e kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-78590-predictor-599b7b7f46-8r44l.18be7ea8fad18a82 namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: c9d35ce4-97d6-427b-b79f-a29f0c36a081 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78590-predictor-599b7b7f46-8r44l namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 24a1816a-ea79-42c5-af42-a3f922f1526e kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-78590-predictor-599b7b7f46-8r44l.18be7ea8fb451f05 namespace: kserve-ci-e2e-test resourceVersion: "26381" uid: be32f84c-2dfe-4fa5-b5cd-714b7e1145ec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78590-predictor-599b7b7f46-8r44l namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 24a1816a-ea79-42c5-af42-a3f922f1526e kind: Event lastTimestamp: "2026-07-02T14:25:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:22Z" name: success-200-isvc-78590-predictor-599b7b7f46-8r44l.18be7ea8fce693a5 namespace: kserve-ci-e2e-test resourceVersion: "30474" uid: 1a9da50b-1b77-48cf-b197-1bce61e30799 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-78590-predictor-599b7b7f46-8r44l namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 24a1816a-ea79-42c5-af42-a3f922f1526e kind: Event lastTimestamp: "2026-07-02T14:25:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:20Z" name: success-200-isvc-78590-predictor-599b7b7f46-8r44l.18be7f2a065e5d3d namespace: kserve-ci-e2e-test resourceVersion: "30444" uid: b3c84520-7531-482b-b767-4f9aab438af2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78590-predictor-599b7b7f46-8r44l namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 24a1816a-ea79-42c5-af42-a3f922f1526e kind: Event lastTimestamp: "2026-07-02T14:25:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:25:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:20Z" name: success-200-isvc-78590-predictor-599b7b7f46-8r44l.18be7f2a065ef218 namespace: kserve-ci-e2e-test resourceVersion: "30446" uid: 2f72d94a-e280-4785-a7f2-8ce2848bbbec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:25:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-78590-predictor-599b7b7f46-8r44l namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: 24a1816a-ea79-42c5-af42-a3f922f1526e kind: Event lastTimestamp: "2026-07-02T14:25:22Z" message: 'Readiness probe failed: Get "https://10.134.0.49:8643/healthz": dial tcp 10.134.0.49:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:25:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:25:22Z" name: success-200-isvc-78590-predictor-599b7b7f46-8r44l.18be7f2a715b6857 namespace: kserve-ci-e2e-test resourceVersion: "30473" uid: 6775b179-ccff-4c91-807d-d1be49c2eee2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-78590-predictor-599b7b7f46 namespace: kserve-ci-e2e-test resourceVersion: "26291" uid: 1beabe2b-95d2-4e62-881e-ca1651c9582b kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: 'Created pod: success-200-isvc-78590-predictor-599b7b7f46-8r44l' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-78590-predictor-599b7b7f46.18be7ea8d8c7572e namespace: kserve-ci-e2e-test resourceVersion: "26309" uid: 9a3c4fe0-3ae9-41ad-a4c5-421c253ef4dd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-78590-predictor namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 3072cdf8-b06f-47c3-b561-ddb8540834cd kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: Scaled up replica set success-200-isvc-78590-predictor-599b7b7f46 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-78590-predictor.18be7ea8d68757d2 namespace: kserve-ci-e2e-test resourceVersion: "26294" uid: 1a2ee709-67be-46c1-8b0c-778d3476185e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:16:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78590-predictor namespace: kserve-ci-e2e-test resourceVersion: "26290" uid: 7148b18a-ecc2-4831-9ba7-1f7afa385ed8 kind: Event lastTimestamp: "2026-07-02T14:17:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:16:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:06Z" name: success-200-isvc-78590-predictor.18be7eac54fdcbdb namespace: kserve-ci-e2e-test resourceVersion: "26967" uid: 57df961c-7360-4071-a3db-dd34fa20df12 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T14:16:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-78590-predictor namespace: kserve-ci-e2e-test resourceVersion: "26290" uid: 7148b18a-ecc2-4831-9ba7-1f7afa385ed8 kind: Event lastTimestamp: "2026-07-02T14:17:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:16:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:06Z" name: success-200-isvc-78590-predictor.18be7eac556cb89d namespace: kserve-ci-e2e-test resourceVersion: "26970" uid: d58ccb74-7873-4ec8-a43f-7def80ad86fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78590 namespace: kserve-ci-e2e-test resourceVersion: "26277" uid: f453b727-ba58-4bb3-97b4-18f433d7cac3 kind: Event lastTimestamp: "2026-07-02T14:16:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:16:52Z" name: success-200-isvc-78590.18be7ea8d68fb029 namespace: kserve-ci-e2e-test resourceVersion: "26843" uid: 412ea4e9-6f28-45f0-b13f-687f9f9e96c5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78590 namespace: kserve-ci-e2e-test resourceVersion: "26277" uid: f453b727-ba58-4bb3-97b4-18f433d7cac3 kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: 'Failed to update status for InferenceService "success-200-isvc-78590": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-78590": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-78590.18be7ea8d779b0b0 namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 3c69cb59-e2d8-457f-aac9-6ce498f839cf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78590 namespace: kserve-ci-e2e-test resourceVersion: "26277" uid: f453b727-ba58-4bb3-97b4-18f433d7cac3 kind: Event lastTimestamp: "2026-07-02T14:16:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-78590": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:16:06Z" name: success-200-isvc-78590.18be7ea8d779fe2c namespace: kserve-ci-e2e-test resourceVersion: "26305" uid: fd11a1d6-ebc3-4c38-81f5-18e039d26aec reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-78590 namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: f453b727-ba58-4bb3-97b4-18f433d7cac3 kind: Event lastTimestamp: "2026-07-02T14:16:52Z" message: InferenceService [success-200-isvc-78590] is Ready metadata: creationTimestamp: "2026-07-02T14:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:16:52Z" name: success-200-isvc-78590.18be7eb3b95fd5b3 namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 475d2a9f-25e3-44f8-8423-a8b0adbfc89e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:05:40.853331Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx namespace: kserve-ci-e2e-test resourceVersion: "20786" uid: 285570f8-0db8-42d5-8404-1e07570d46a0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:05:40Z" name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx.18be7e1742adfcb1 namespace: kserve-ci-e2e-test resourceVersion: "20795" uid: c1fa5a22-0d29-42df-b430-a6f81ad90fc0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 285570f8-0db8-42d5-8404-1e07570d46a0 kind: Event lastTimestamp: "2026-07-02T14:05:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7b7ab-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:40Z" name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx.18be7e174a3813c6 namespace: kserve-ci-e2e-test resourceVersion: "20832" uid: 16c57a36-4379-4fba-8d75-9633171d2d66 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: 285570f8-0db8-42d5-8404-1e07570d46a0 kind: Event lastTimestamp: "2026-07-02T14:05:41Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:05:41Z" name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx.18be7e177fff4555 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 59582228-4e9f-4f25-8307-3f5f2ce74094 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 285570f8-0db8-42d5-8404-1e07570d46a0 kind: Event lastTimestamp: "2026-07-02T14:05:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:41Z" name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx.18be7e17815de060 namespace: kserve-ci-e2e-test resourceVersion: "20857" uid: 3aec08ab-8302-4da5-af59-24f5e6798549 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 285570f8-0db8-42d5-8404-1e07570d46a0 kind: Event lastTimestamp: "2026-07-02T14:05:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:41Z" name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx.18be7e1784d262fd namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: c5e2ddaa-0a4a-4d42-bde9-2e102be36cd3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 285570f8-0db8-42d5-8404-1e07570d46a0 kind: Event lastTimestamp: "2026-07-02T14:05:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:41Z" name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx.18be7e17853e1cb7 namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: e09263b0-6d7c-4230-acf8-2a02e7f1b25f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 285570f8-0db8-42d5-8404-1e07570d46a0 kind: Event lastTimestamp: "2026-07-02T14:05:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:41Z" name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx.18be7e178547a1de namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: 4c5f1796-1a38-45ca-98d0-ae49f0d0deea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 285570f8-0db8-42d5-8404-1e07570d46a0 kind: Event lastTimestamp: "2026-07-02T14:05:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:42Z" name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx.18be7e1788580122 namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: 52575a74-7bb5-4655-ab0c-fad9e063b2b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 285570f8-0db8-42d5-8404-1e07570d46a0 kind: Event lastTimestamp: "2026-07-02T14:05:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:42Z" name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx.18be7e1788d56923 namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: da8a889a-ab34-4229-b046-f97459d50929 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:05:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 285570f8-0db8-42d5-8404-1e07570d46a0 kind: Event lastTimestamp: "2026-07-02T14:06:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:05:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:18Z" name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx.18be7e17c0a5be7a namespace: kserve-ci-e2e-test resourceVersion: "21209" uid: c425ef0c-50e4-4d4a-9c71-6780d0bcd4f4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 285570f8-0db8-42d5-8404-1e07570d46a0 kind: Event lastTimestamp: "2026-07-02T14:14:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:55Z" name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx.18be7e987df4622d namespace: kserve-ci-e2e-test resourceVersion: "25152" uid: a3446ed2-9987-4178-8a87-c22ba399c79a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:14:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx namespace: kserve-ci-e2e-test resourceVersion: "20792" uid: 285570f8-0db8-42d5-8404-1e07570d46a0 kind: Event lastTimestamp: "2026-07-02T14:14:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:14:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:14:55Z" name: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx.18be7e987df59032 namespace: kserve-ci-e2e-test resourceVersion: "25155" uid: df4f0aed-3438-4d16-88d8-e9b4a8e37140 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7b7ab-predictor-6cbf754778 namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: 5c8cb45f-3cf2-438a-8b14-4087c1aa8100 kind: Event lastTimestamp: "2026-07-02T14:05:40Z" message: 'Created pod: success-200-isvc-7b7ab-predictor-6cbf754778-jfbcx' metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:40Z" name: success-200-isvc-7b7ab-predictor-6cbf754778.18be7e17421cdc06 namespace: kserve-ci-e2e-test resourceVersion: "20790" uid: 5324c24d-a0c4-403e-86b1-9ec600073ef1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7b7ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: cdcaee46-b8f8-4e63-8de6-8c216db2c141 kind: Event lastTimestamp: "2026-07-02T14:05:40Z" message: Scaled up replica set success-200-isvc-7b7ab-predictor-6cbf754778 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:40Z" name: success-200-isvc-7b7ab-predictor.18be7e174015b574 namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: 209a6ace-6c59-492c-8938-cab3150ab664 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b7ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 56ea3244-d39c-40d9-9109-5ae953314010 kind: Event lastTimestamp: "2026-07-02T14:06:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:10Z" name: success-200-isvc-7b7ab-predictor.18be7e1abea237cc namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: 9fe0fcca-72bd-43a3-8d93-67d9c21656a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b7ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: 56ea3244-d39c-40d9-9109-5ae953314010 kind: Event lastTimestamp: "2026-07-02T14:06:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:10Z" name: success-200-isvc-7b7ab-predictor.18be7e1abf0d0bad namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: e9f00cc4-c855-49cf-b065-6bbf9e34fb24 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:06:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b7ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: 56ea3244-d39c-40d9-9109-5ae953314010 kind: Event lastTimestamp: "2026-07-02T14:06:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:55Z" name: success-200-isvc-7b7ab-predictor.18be7e21bd6249b8 namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: 5b6e0d7a-df93-4f9b-b3e7-929069f3c5b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:06:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7b7ab-predictor namespace: kserve-ci-e2e-test resourceVersion: "21000" uid: 56ea3244-d39c-40d9-9109-5ae953314010 kind: Event lastTimestamp: "2026-07-02T14:06:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:06:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:55Z" name: success-200-isvc-7b7ab-predictor.18be7e21bdd5c8ef namespace: kserve-ci-e2e-test resourceVersion: "21734" uid: 07c306ff-a8db-4df8-8675-c809bcb0a7c6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7b7ab namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: 5ee5b4f7-8f0a-4878-a305-0ccab872a449 kind: Event lastTimestamp: "2026-07-02T14:06:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:29Z" name: success-200-isvc-7b7ab.18be7e17401ff6d3 namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: 1bcc796d-e8fb-4f7c-92a6-4a814a495663 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7b7ab namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: 5ee5b4f7-8f0a-4878-a305-0ccab872a449 kind: Event lastTimestamp: "2026-07-02T14:06:29Z" message: InferenceService [success-200-isvc-7b7ab] is Ready metadata: creationTimestamp: "2026-07-02T14:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:29Z" name: success-200-isvc-7b7ab.18be7e227c4dfc3c namespace: kserve-ci-e2e-test resourceVersion: "21327" uid: 6de8e736-34f5-4a9e-9942-64a95f73899d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:56:11.876709Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-88827-predictor-68bb654446-xn6vz namespace: kserve-ci-e2e-test resourceVersion: "16175" uid: 71af5689-176e-4d90-8d89-d5a70a978f78 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-88827-predictor-68bb654446-xn6vz to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:56:11Z" name: success-200-isvc-88827-predictor-68bb654446-xn6vz.18be7d92c90843b2 namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: 8bf18d4f-a9da-410f-8107-3eb48c873a6e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-88827-predictor-68bb654446-xn6vz namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 71af5689-176e-4d90-8d89-d5a70a978f78 kind: Event lastTimestamp: "2026-07-02T13:56:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-88827-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:12Z" name: success-200-isvc-88827-predictor-68bb654446-xn6vz.18be7d92d5e506a3 namespace: kserve-ci-e2e-test resourceVersion: "16249" uid: bc8ca14e-368d-492f-9e54-671fa2651a1e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-88827-predictor-68bb654446-xn6vz namespace: kserve-ci-e2e-test resourceVersion: "16198" uid: 71af5689-176e-4d90-8d89-d5a70a978f78 kind: Event lastTimestamp: "2026-07-02T13:56:12Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:56:12Z" name: success-200-isvc-88827-predictor-68bb654446-xn6vz.18be7d93066a91c5 namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: 644ed733-71d8-45aa-9ed9-8e28eaae0b48 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-88827-predictor-68bb654446-xn6vz namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 71af5689-176e-4d90-8d89-d5a70a978f78 kind: Event lastTimestamp: "2026-07-02T13:56:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T13:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:12Z" name: success-200-isvc-88827-predictor-68bb654446-xn6vz.18be7d9307dca588 namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: 82dfc9ff-e688-4483-83e4-7e30394fcbae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-88827-predictor-68bb654446-xn6vz namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 71af5689-176e-4d90-8d89-d5a70a978f78 kind: Event lastTimestamp: "2026-07-02T13:56:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:12Z" name: success-200-isvc-88827-predictor-68bb654446-xn6vz.18be7d930a782dc3 namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: 68abc622-d094-43b7-af72-50fb2a742cd0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-88827-predictor-68bb654446-xn6vz namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 71af5689-176e-4d90-8d89-d5a70a978f78 kind: Event lastTimestamp: "2026-07-02T13:56:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:12Z" name: success-200-isvc-88827-predictor-68bb654446-xn6vz.18be7d930adaea9d namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 3bb25366-3881-455e-8213-459ecf879217 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-88827-predictor-68bb654446-xn6vz namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 71af5689-176e-4d90-8d89-d5a70a978f78 kind: Event lastTimestamp: "2026-07-02T13:56:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:56:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:12Z" name: success-200-isvc-88827-predictor-68bb654446-xn6vz.18be7d930ae40931 namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: e89ca728-b972-4a86-86a1-34f4eff653eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-88827-predictor-68bb654446-xn6vz namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 71af5689-176e-4d90-8d89-d5a70a978f78 kind: Event lastTimestamp: "2026-07-02T13:56:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:13Z" name: success-200-isvc-88827-predictor-68bb654446-xn6vz.18be7d930d8321e5 namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: 45c34c50-17fb-49d9-b234-fa2410ace93f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-88827-predictor-68bb654446-xn6vz namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 71af5689-176e-4d90-8d89-d5a70a978f78 kind: Event lastTimestamp: "2026-07-02T13:56:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:13Z" name: success-200-isvc-88827-predictor-68bb654446-xn6vz.18be7d930de79844 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: df4331e4-cef6-4724-a7cb-0f0a9c598d84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:56:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-88827-predictor-68bb654446-xn6vz namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 71af5689-176e-4d90-8d89-d5a70a978f78 kind: Event lastTimestamp: "2026-07-02T14:05:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:29Z" name: success-200-isvc-88827-predictor-68bb654446-xn6vz.18be7d9325efa6e6 namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: c2252b89-d131-4959-85f4-bf8bdc0b287b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-88827-predictor-68bb654446-xn6vz namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 71af5689-176e-4d90-8d89-d5a70a978f78 kind: Event lastTimestamp: "2026-07-02T14:05:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:26Z" name: success-200-isvc-88827-predictor-68bb654446-xn6vz.18be7e13fada3d70 namespace: kserve-ci-e2e-test resourceVersion: "20494" uid: 7f21d1b4-ca95-4ff3-9bc5-e9bc719ed562 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-88827-predictor-68bb654446-xn6vz namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 71af5689-176e-4d90-8d89-d5a70a978f78 kind: Event lastTimestamp: "2026-07-02T14:05:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:26Z" name: success-200-isvc-88827-predictor-68bb654446-xn6vz.18be7e13fadb1058 namespace: kserve-ci-e2e-test resourceVersion: "20500" uid: 8bc9be0f-da95-4ac2-98b6-ed9089ed4ac1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-88827-predictor-68bb654446-xn6vz namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: 71af5689-176e-4d90-8d89-d5a70a978f78 kind: Event lastTimestamp: "2026-07-02T14:05:29Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:29Z" name: success-200-isvc-88827-predictor-68bb654446-xn6vz.18be7e149a57e7ab namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: 4c0b3cf2-f1b1-4caa-993a-b0acd2fca888 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-88827-predictor-68bb654446 namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: 6debdc54-6c2b-467f-9335-e0e17343f05e kind: Event lastTimestamp: "2026-07-02T13:56:11Z" message: 'Created pod: success-200-isvc-88827-predictor-68bb654446-xn6vz' metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:11Z" name: success-200-isvc-88827-predictor-68bb654446.18be7d92c87f54ec namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: c721981b-a7d5-42aa-8ace-5b6890f77200 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-88827-predictor namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: 66e7dd04-c3f8-49b4-ac9f-34d8b0e5fd00 kind: Event lastTimestamp: "2026-07-02T13:56:11Z" message: Scaled up replica set success-200-isvc-88827-predictor-68bb654446 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:11Z" name: success-200-isvc-88827-predictor.18be7d92c6aadf9d namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 624f2af9-a3c2-4190-863e-81e91b788f2f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:56:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-88827-predictor namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: bb31a046-c203-4fe5-9df6-e131966fcf86 kind: Event lastTimestamp: "2026-07-02T13:56:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:56:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:41Z" name: success-200-isvc-88827-predictor.18be7d9645b4b2e1 namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: c076e50b-64c6-46b7-8e47-d7b9d1b553fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:56:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-88827-predictor namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: bb31a046-c203-4fe5-9df6-e131966fcf86 kind: Event lastTimestamp: "2026-07-02T13:56:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:56:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:41Z" name: success-200-isvc-88827-predictor.18be7d96465b0ae5 namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: f78ab803-37eb-47a7-857d-3b9945f78359 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:56:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-88827-predictor namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: bb31a046-c203-4fe5-9df6-e131966fcf86 kind: Event lastTimestamp: "2026-07-02T13:57:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:26Z" name: success-200-isvc-88827-predictor.18be7d9d44d5746e namespace: kserve-ci-e2e-test resourceVersion: "16966" uid: 692b2a9f-ad8e-4638-a26b-c5baa8f84cd5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:56:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-88827-predictor namespace: kserve-ci-e2e-test resourceVersion: "16452" uid: bb31a046-c203-4fe5-9df6-e131966fcf86 kind: Event lastTimestamp: "2026-07-02T13:57:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:57:26Z" name: success-200-isvc-88827-predictor.18be7d9d45254a28 namespace: kserve-ci-e2e-test resourceVersion: "16967" uid: 0df6d1b6-8a6e-41f0-9b03-6eb582b2b659 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:56:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-88827 namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: 914aae4b-90d4-4c5e-9047-c496bfb7d95c kind: Event lastTimestamp: "2026-07-02T13:56:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:56:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:56:59Z" name: success-200-isvc-88827.18be7d92c6ec2c52 namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 7bf5dc58-ac39-4e1d-808f-b28561eceef1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-88827 namespace: kserve-ci-e2e-test resourceVersion: "16721" uid: 914aae4b-90d4-4c5e-9047-c496bfb7d95c kind: Event lastTimestamp: "2026-07-02T13:56:59Z" message: InferenceService [success-200-isvc-88827] is Ready metadata: creationTimestamp: "2026-07-02T13:56:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:56:59Z" name: success-200-isvc-88827.18be7d9de516a9ab namespace: kserve-ci-e2e-test resourceVersion: "16722" uid: ebaa4b20-9c98-4df8-89f0-6be15235dedb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:15:51.851519Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8 namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: f294d0b1-01b6-40de-aef0-c6cdb40552a3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8 to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:15:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:15:51Z" name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8.18be7ea58502778d namespace: kserve-ci-e2e-test resourceVersion: "26018" uid: 17d03b89-cbb3-4607-b33c-ba07de5317d1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8 namespace: kserve-ci-e2e-test resourceVersion: "26026" uid: f294d0b1-01b6-40de-aef0-c6cdb40552a3 kind: Event lastTimestamp: "2026-07-02T14:15:52Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:15:52Z" name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8.18be7ea59eba6e67 namespace: kserve-ci-e2e-test resourceVersion: "26075" uid: 0c2e0f1b-c503-4547-abd8-dd4b0e6b3285 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8 namespace: kserve-ci-e2e-test resourceVersion: "26016" uid: f294d0b1-01b6-40de-aef0-c6cdb40552a3 kind: Event lastTimestamp: "2026-07-02T14:15:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:52Z" name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8.18be7ea5a02cc382 namespace: kserve-ci-e2e-test resourceVersion: "26077" uid: 71ea8d1d-ee53-49d6-8316-2a132ffdec02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8 namespace: kserve-ci-e2e-test resourceVersion: "26016" uid: f294d0b1-01b6-40de-aef0-c6cdb40552a3 kind: Event lastTimestamp: "2026-07-02T14:15:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:52Z" name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8.18be7ea5a3f45f2d namespace: kserve-ci-e2e-test resourceVersion: "26082" uid: ab4a8c59-4255-406a-991a-6d9f3bdcb3be reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8 namespace: kserve-ci-e2e-test resourceVersion: "26016" uid: f294d0b1-01b6-40de-aef0-c6cdb40552a3 kind: Event lastTimestamp: "2026-07-02T14:15:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:52Z" name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8.18be7ea5a4650baf namespace: kserve-ci-e2e-test resourceVersion: "26083" uid: 04b9806d-239c-4977-8797-e440f47d8536 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8 namespace: kserve-ci-e2e-test resourceVersion: "26016" uid: f294d0b1-01b6-40de-aef0-c6cdb40552a3 kind: Event lastTimestamp: "2026-07-02T14:15:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:52Z" name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8.18be7ea5a46e8c56 namespace: kserve-ci-e2e-test resourceVersion: "26084" uid: 67d46c9b-d1d1-40a0-9f67-79af1860e68f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8 namespace: kserve-ci-e2e-test resourceVersion: "26016" uid: f294d0b1-01b6-40de-aef0-c6cdb40552a3 kind: Event lastTimestamp: "2026-07-02T14:15:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:52Z" name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8.18be7ea5a7aae4d9 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: 52d4bf1e-72f5-41e1-9d7c-e9904212e766 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8 namespace: kserve-ci-e2e-test resourceVersion: "26016" uid: f294d0b1-01b6-40de-aef0-c6cdb40552a3 kind: Event lastTimestamp: "2026-07-02T14:15:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:15:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:15:52Z" name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8.18be7ea5a839b088 namespace: kserve-ci-e2e-test resourceVersion: "26087" uid: e59bb272-e115-4138-b6e4-103fa1dd96ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:15:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8 namespace: kserve-ci-e2e-test resourceVersion: "26016" uid: f294d0b1-01b6-40de-aef0-c6cdb40552a3 kind: Event lastTimestamp: "2026-07-02T14:16:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:15:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:29Z" name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8.18be7ea5f1b50145 namespace: kserve-ci-e2e-test resourceVersion: "26606" uid: d323cbea-970b-4085-bc11-e5b4c50a19c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:33:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8 namespace: kserve-ci-e2e-test resourceVersion: "26016" uid: f294d0b1-01b6-40de-aef0-c6cdb40552a3 kind: Event lastTimestamp: "2026-07-02T14:33:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:33:11Z" name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8.18be7f979b7cd8ae namespace: kserve-ci-e2e-test resourceVersion: "33698" uid: dd475c3a-5ef9-4949-8a5d-66fbf0012f5a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:33:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8 namespace: kserve-ci-e2e-test resourceVersion: "26016" uid: f294d0b1-01b6-40de-aef0-c6cdb40552a3 kind: Event lastTimestamp: "2026-07-02T14:33:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:33:11Z" name: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8.18be7f979b7df540 namespace: kserve-ci-e2e-test resourceVersion: "33699" uid: 9d8c73dd-8e7a-4c72-a9ed-a990a69472af reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8d2ee-predictor-5d9c964d96 namespace: kserve-ci-e2e-test resourceVersion: "25997" uid: 2236a808-62d2-46cf-87c5-88bdc05d45c8 kind: Event lastTimestamp: "2026-07-02T14:15:51Z" message: 'Created pod: success-200-isvc-8d2ee-predictor-5d9c964d96-7txl8' metadata: creationTimestamp: "2026-07-02T14:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:51Z" name: success-200-isvc-8d2ee-predictor-5d9c964d96.18be7ea5845cc169 namespace: kserve-ci-e2e-test resourceVersion: "26015" uid: 65fa2f63-6114-4538-8283-794771b7b846 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:15:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8d2ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "25990" uid: 3dcd1129-2254-410b-9474-f8e976ba4e67 kind: Event lastTimestamp: "2026-07-02T14:15:51Z" message: Scaled up replica set success-200-isvc-8d2ee-predictor-5d9c964d96 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:15:51Z" name: success-200-isvc-8d2ee-predictor.18be7ea5819ad7f6 namespace: kserve-ci-e2e-test resourceVersion: "26006" uid: 5ccf7084-f29f-40cd-a948-990b3c2dbb50 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d2ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: ef459dff-4e60-4e74-a3e8-d19489dee2da kind: Event lastTimestamp: "2026-07-02T14:16:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:16:36Z" name: success-200-isvc-8d2ee-predictor.18be7ea900d461c2 namespace: kserve-ci-e2e-test resourceVersion: "26667" uid: 3f469c29-c79d-4bea-b2ef-35f8e2788f2f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T14:16:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d2ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: ef459dff-4e60-4e74-a3e8-d19489dee2da kind: Event lastTimestamp: "2026-07-02T14:16:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:16:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:16:36Z" name: success-200-isvc-8d2ee-predictor.18be7ea9014a55f9 namespace: kserve-ci-e2e-test resourceVersion: "26668" uid: 2bc3a31f-a7ab-47c8-8417-93114c063063 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:16:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d2ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26392" uid: ef459dff-4e60-4e74-a3e8-d19489dee2da kind: Event lastTimestamp: "2026-07-02T14:17:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:06Z" name: success-200-isvc-8d2ee-predictor.18be7eb37e1e971d namespace: kserve-ci-e2e-test resourceVersion: "26997" uid: 991bb7ee-d820-462a-9962-d226ad2f2212 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:16:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d2ee-predictor namespace: kserve-ci-e2e-test resourceVersion: "26392" uid: ef459dff-4e60-4e74-a3e8-d19489dee2da kind: Event lastTimestamp: "2026-07-02T14:17:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:06Z" name: success-200-isvc-8d2ee-predictor.18be7eb37eaac992 namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: 720b5ea8-5735-495e-8b35-081878166a76 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T14:15:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d2ee namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 3feb7e7a-c74e-41e7-8ff2-7b51f9e1c0aa kind: Event lastTimestamp: "2026-07-02T14:16:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:15:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:16:39Z" name: success-200-isvc-8d2ee.18be7ea581ba7f73 namespace: kserve-ci-e2e-test resourceVersion: "26711" uid: b11f5734-3df1-4a5a-9ff1-b59c7064e5a1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d2ee namespace: kserve-ci-e2e-test resourceVersion: "26709" uid: 3feb7e7a-c74e-41e7-8ff2-7b51f9e1c0aa kind: Event lastTimestamp: "2026-07-02T14:16:39Z" message: InferenceService [success-200-isvc-8d2ee] is Ready metadata: creationTimestamp: "2026-07-02T14:16:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:16:39Z" name: success-200-isvc-8d2ee.18be7eb0aeac61b9 namespace: kserve-ci-e2e-test resourceVersion: "26710" uid: 3f0274ef-0863-46fb-b48c-500e6a51c8ff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:05:26.770739Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c559a-predictor-665765c694-bj667 namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 915650ad-3f44-487c-802d-23748fce727c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c559a-predictor-665765c694-bj667 to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:05:26Z" name: success-200-isvc-c559a-predictor-665765c694-bj667.18be7e13fb4a9f35 namespace: kserve-ci-e2e-test resourceVersion: "20499" uid: 1b5b1125-d904-4937-b203-9667176cb3ce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c559a-predictor-665765c694-bj667 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 915650ad-3f44-487c-802d-23748fce727c kind: Event lastTimestamp: "2026-07-02T14:05:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c559a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:26Z" name: success-200-isvc-c559a-predictor-665765c694-bj667.18be7e1406a9876e namespace: kserve-ci-e2e-test resourceVersion: "20537" uid: 5dc7429b-2029-4e79-a9e4-e687302e390d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c559a-predictor-665765c694-bj667 namespace: kserve-ci-e2e-test resourceVersion: "20505" uid: 915650ad-3f44-487c-802d-23748fce727c kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:05:27Z" name: success-200-isvc-c559a-predictor-665765c694-bj667.18be7e14391f6a62 namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 175d8954-578f-4619-b225-256fb76703b2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c559a-predictor-665765c694-bj667 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 915650ad-3f44-487c-802d-23748fce727c kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:27Z" name: success-200-isvc-c559a-predictor-665765c694-bj667.18be7e143a95d18e namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: dfc84043-67be-4efd-9ba7-c13fe3fd35c0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c559a-predictor-665765c694-bj667 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 915650ad-3f44-487c-802d-23748fce727c kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:27Z" name: success-200-isvc-c559a-predictor-665765c694-bj667.18be7e143d594df0 namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: 61b9d022-0595-4907-97e2-7219f051f507 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c559a-predictor-665765c694-bj667 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 915650ad-3f44-487c-802d-23748fce727c kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:27Z" name: success-200-isvc-c559a-predictor-665765c694-bj667.18be7e143dda0965 namespace: kserve-ci-e2e-test resourceVersion: "20563" uid: 6ee3d4da-e73d-4370-844d-4a6d8da9319e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c559a-predictor-665765c694-bj667 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 915650ad-3f44-487c-802d-23748fce727c kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:27Z" name: success-200-isvc-c559a-predictor-665765c694-bj667.18be7e143de31a8c namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: 4b9a052d-6915-4b80-acc3-aa0eb6db6415 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c559a-predictor-665765c694-bj667 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 915650ad-3f44-487c-802d-23748fce727c kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:27Z" name: success-200-isvc-c559a-predictor-665765c694-bj667.18be7e1440ee2b73 namespace: kserve-ci-e2e-test resourceVersion: "20566" uid: 019f5627-6f44-411e-be90-2219705fb80a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c559a-predictor-665765c694-bj667 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 915650ad-3f44-487c-802d-23748fce727c kind: Event lastTimestamp: "2026-07-02T14:05:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:05:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:27Z" name: success-200-isvc-c559a-predictor-665765c694-bj667.18be7e14416022f7 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 5225cf71-ceee-4479-8464-262b41168083 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:05:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c559a-predictor-665765c694-bj667 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 915650ad-3f44-487c-802d-23748fce727c kind: Event lastTimestamp: "2026-07-02T14:06:05Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:05:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:05Z" name: success-200-isvc-c559a-predictor-665765c694-bj667.18be7e14b5db2d1b namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: aa8e0184-21b8-49cd-9dbf-28bda75a7e75 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c559a-predictor-665765c694-bj667 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 915650ad-3f44-487c-802d-23748fce727c kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:37Z" name: success-200-isvc-c559a-predictor-665765c694-bj667.18be7e245600037c namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: 4ff0baab-3301-4af2-b731-2c15204679fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c559a-predictor-665765c694-bj667 namespace: kserve-ci-e2e-test resourceVersion: "20496" uid: 915650ad-3f44-487c-802d-23748fce727c kind: Event lastTimestamp: "2026-07-02T14:06:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:06:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:06:37Z" name: success-200-isvc-c559a-predictor-665765c694-bj667.18be7e245600679f namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 7e3e0502-475a-4beb-9a17-da0e420c3eb0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c559a-predictor-665765c694 namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 4d078e5f-8afe-42d7-85bd-54f397cf549e kind: Event lastTimestamp: "2026-07-02T14:05:26Z" message: 'Created pod: success-200-isvc-c559a-predictor-665765c694-bj667' metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:26Z" name: success-200-isvc-c559a-predictor-665765c694.18be7e13fad992ab namespace: kserve-ci-e2e-test resourceVersion: "20495" uid: 6a767d5f-0234-4f03-be50-41a33d5ed4d0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c559a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20468" uid: 7a039370-2593-449d-8eb5-27c6918d3542 kind: Event lastTimestamp: "2026-07-02T14:05:26Z" message: Scaled up replica set success-200-isvc-c559a-predictor-665765c694 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:26Z" name: success-200-isvc-c559a-predictor.18be7e13f899409b namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: d1658f97-41d3-4489-a9b7-d6b2da414b05 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c559a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: edbd6497-06db-46d0-af5e-7decbc5f456a kind: Event lastTimestamp: "2026-07-02T14:05:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:56Z" name: success-200-isvc-c559a-predictor.18be7e17775dabc0 namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 15aeaeef-3451-4fdd-af4b-6f73c33d384a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c559a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20472" uid: edbd6497-06db-46d0-af5e-7decbc5f456a kind: Event lastTimestamp: "2026-07-02T14:05:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:05:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:56Z" name: success-200-isvc-c559a-predictor.18be7e1777d15701 namespace: kserve-ci-e2e-test resourceVersion: "21013" uid: 70f6abbb-5702-4926-9adb-a0c5386b3b18 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:06:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c559a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: edbd6497-06db-46d0-af5e-7decbc5f456a kind: Event lastTimestamp: "2026-07-02T14:06:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:26Z" name: success-200-isvc-c559a-predictor.18be7e1e753db376 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: bbb23bd2-2fc9-4672-9279-429475810439 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:06:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c559a-predictor namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: edbd6497-06db-46d0-af5e-7decbc5f456a kind: Event lastTimestamp: "2026-07-02T14:06:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:06:26Z" name: success-200-isvc-c559a-predictor.18be7e1e759fa16b namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: 606d068e-e1a3-438e-a9e8-1ca5d7adbb1d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:05:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c559a namespace: kserve-ci-e2e-test resourceVersion: "20449" uid: 3be6f37f-36f7-4c21-8481-bc01bfc4e486 kind: Event lastTimestamp: "2026-07-02T14:06:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:05:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:16Z" name: success-200-isvc-c559a.18be7e13f8c48a22 namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: 7970b4e2-1b35-4b07-abb1-b2606ea9a2fe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:06:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c559a namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: 3be6f37f-36f7-4c21-8481-bc01bfc4e486 kind: Event lastTimestamp: "2026-07-02T14:06:15Z" message: InferenceService [success-200-isvc-c559a] is Ready metadata: creationTimestamp: "2026-07-02T14:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:06:15Z" name: success-200-isvc-c559a.18be7e1f70f6fff3 namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: fed36c96-f272-42b7-912f-84a2350e3b3f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:04:30.588276Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: cecfa45c-56ff-4bf4-a639-530b29bb9418 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c675b-predictor-7b8c7b5778-542pz to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:04:30Z" name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz.18be7e06e68e5261 namespace: kserve-ci-e2e-test resourceVersion: "19907" uid: a69ed0a2-7c88-49cc-9811-2e5a0caab81a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: cecfa45c-56ff-4bf4-a639-530b29bb9418 kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c675b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:30Z" name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz.18be7e06f2f56e88 namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: d373c70b-0707-49d1-ada7-181edd3ce973 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: cecfa45c-56ff-4bf4-a639-530b29bb9418 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:04:31Z" name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz.18be7e0724dc1bcd namespace: kserve-ci-e2e-test resourceVersion: "19971" uid: 8d9bd3d3-ecef-4f9f-b187-e4197354a7ac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: cecfa45c-56ff-4bf4-a639-530b29bb9418 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1695" already present on machine metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:31Z" name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz.18be7e07263d0135 namespace: kserve-ci-e2e-test resourceVersion: "19973" uid: 1403d6f7-ba09-4689-adb1-cc82e7c0671d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: cecfa45c-56ff-4bf4-a639-530b29bb9418 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:31Z" name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz.18be7e07298ed707 namespace: kserve-ci-e2e-test resourceVersion: "19976" uid: e74874b1-4149-4b1f-bcd4-f3deb0aff7c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: cecfa45c-56ff-4bf4-a639-530b29bb9418 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:31Z" name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz.18be7e0729ffed3d namespace: kserve-ci-e2e-test resourceVersion: "19978" uid: a96f8ce6-2e6b-4787-a9f0-d6f4b1159343 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: cecfa45c-56ff-4bf4-a639-530b29bb9418 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:31Z" name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz.18be7e072a0ae1fb namespace: kserve-ci-e2e-test resourceVersion: "19979" uid: 6fe7b46c-b289-4fb7-aa5d-04fd9e0fa575 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: cecfa45c-56ff-4bf4-a639-530b29bb9418 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:31Z" name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz.18be7e072d513f43 namespace: kserve-ci-e2e-test resourceVersion: "19982" uid: 6a2d3c21-23e7-4228-98ba-e3e861d6c90f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: cecfa45c-56ff-4bf4-a639-530b29bb9418 kind: Event lastTimestamp: "2026-07-02T14:04:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:04:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:31Z" name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz.18be7e072dc0f994 namespace: kserve-ci-e2e-test resourceVersion: "19984" uid: 916d4cb9-882f-4a90-8a22-70c12b3ee024 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:04:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: cecfa45c-56ff-4bf4-a639-530b29bb9418 kind: Event lastTimestamp: "2026-07-02T14:05:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:04:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:08Z" name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz.18be7e076091e33f namespace: kserve-ci-e2e-test resourceVersion: "20277" uid: 2becc970-52f0-4ca2-adcd-bdd9506c27ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: cecfa45c-56ff-4bf4-a639-530b29bb9418 kind: Event lastTimestamp: "2026-07-02T14:05:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:40Z" name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz.18be7e174213e442 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 39c5aa5a-4401-4023-a195-8d16818f8a5f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: cecfa45c-56ff-4bf4-a639-530b29bb9418 kind: Event lastTimestamp: "2026-07-02T14:05:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T14:05:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:40Z" name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz.18be7e17421474e1 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: 2f92a280-50c3-4d26-a184-80329c473f5c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz namespace: kserve-ci-e2e-test resourceVersion: "19899" uid: cecfa45c-56ff-4bf4-a639-530b29bb9418 kind: Event lastTimestamp: "2026-07-02T14:05:43Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T14:05:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:05:43Z" name: success-200-isvc-c675b-predictor-7b8c7b5778-542pz.18be7e17e8caa1ae namespace: kserve-ci-e2e-test resourceVersion: "20893" uid: 05fde672-c764-4917-b3a3-cd8169a99c87 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c675b-predictor-7b8c7b5778 namespace: kserve-ci-e2e-test resourceVersion: "19878" uid: 2976627b-2305-40e3-9ec2-ec7b52583d47 kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: 'Created pod: success-200-isvc-c675b-predictor-7b8c7b5778-542pz' metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:04:30Z" name: success-200-isvc-c675b-predictor-7b8c7b5778.18be7e06e5f7fdde namespace: kserve-ci-e2e-test resourceVersion: "19898" uid: 76156bc0-bbaa-4379-843e-647c614e6bdb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c675b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19876" uid: 0977de9a-fb00-437c-891e-889d01f989f1 kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: Scaled up replica set success-200-isvc-c675b-predictor-7b8c7b5778 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:04:30Z" name: success-200-isvc-c675b-predictor.18be7e06e271f7ae namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: 5b8f90a7-ec3c-47bb-abb5-d317e959edac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:04:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c675b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19877" uid: 02449607-5fd4-4682-9e31-cbb14d953b84 kind: Event lastTimestamp: "2026-07-02T14:05:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:00Z" name: success-200-isvc-c675b-predictor.18be7e0a61368749 namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: 3102bba3-7c30-4a96-b0be-11d688f17d01 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:04:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c675b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19877" uid: 02449607-5fd4-4682-9e31-cbb14d953b84 kind: Event lastTimestamp: "2026-07-02T14:05:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:04:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:00Z" name: success-200-isvc-c675b-predictor.18be7e0a6189fba5 namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 6ecad747-b6e6-4afd-b8c0-18559c37d33b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c675b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20104" uid: 02449607-5fd4-4682-9e31-cbb14d953b84 kind: Event lastTimestamp: "2026-07-02T14:05:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:30Z" name: success-200-isvc-c675b-predictor.18be7e115ff379bd namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: 499e00fd-48e9-4623-8415-b857a4bdbf31 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:05:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c675b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20104" uid: 02449607-5fd4-4682-9e31-cbb14d953b84 kind: Event lastTimestamp: "2026-07-02T14:05:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T14:05:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:05:30Z" name: success-200-isvc-c675b-predictor.18be7e116063fd4e namespace: kserve-ci-e2e-test resourceVersion: "20615" uid: 8f004808-c949-4e54-8ade-5bfc476e7c8d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c675b namespace: kserve-ci-e2e-test resourceVersion: "19868" uid: 90c6374c-8b63-4963-95aa-6526e981a8a6 kind: Event lastTimestamp: "2026-07-02T14:05:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:18Z" name: success-200-isvc-c675b.18be7e06e27fed54 namespace: kserve-ci-e2e-test resourceVersion: "20372" uid: d85e9e80-810f-493d-9fab-212257e308b2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c675b namespace: kserve-ci-e2e-test resourceVersion: "19868" uid: 90c6374c-8b63-4963-95aa-6526e981a8a6 kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: 'Failed to update status for InferenceService "success-200-isvc-c675b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c675b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:04:30Z" name: success-200-isvc-c675b.18be7e06e30aae75 namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: 73f05e5e-970c-454f-bff6-3cb0f7d76e36 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c675b namespace: kserve-ci-e2e-test resourceVersion: "19868" uid: 90c6374c-8b63-4963-95aa-6526e981a8a6 kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c675b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:04:30Z" name: success-200-isvc-c675b.18be7e06e30afdae namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: ebbc8415-f142-4ec3-a248-e5dbaa90f54b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:05:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c675b namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 90c6374c-8b63-4963-95aa-6526e981a8a6 kind: Event lastTimestamp: "2026-07-02T14:05:18Z" message: InferenceService [success-200-isvc-c675b] is Ready metadata: creationTimestamp: "2026-07-02T14:05:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:05:18Z" name: success-200-isvc-c675b.18be7e121c10ca5d namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 9e934a3c-e070-4245-88cb-eebf27c3181a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:55:01.487472Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-40523-6b6759674-ksx98 namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: 3013f16b-01cc-436e-be2a-691c507712f6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-40523-6b6759674-ksx98 to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T13:55:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:55:01Z" name: switch-graph-40523-6b6759674-ksx98.18be7d8265814b06 namespace: kserve-ci-e2e-test resourceVersion: "15315" uid: 3b5774e6-d8ba-456b-bd5a-12e3972a8482 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:01Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-40523-6b6759674-ksx98 namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: 3013f16b-01cc-436e-be2a-691c507712f6 kind: Event lastTimestamp: "2026-07-02T13:55:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-40523-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:01Z" name: switch-graph-40523-6b6759674-ksx98.18be7d826d1a682e namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: c167a1c5-986f-4ef0-84fa-eae3dbc7701d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:02Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-40523-6b6759674-ksx98 namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 3013f16b-01cc-436e-be2a-691c507712f6 kind: Event lastTimestamp: "2026-07-02T13:55:02Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:55:02Z" name: switch-graph-40523-6b6759674-ksx98.18be7d82a2ab4c65 namespace: kserve-ci-e2e-test resourceVersion: "15334" uid: 069614f6-45f8-408a-936a-f28dbb7718fb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-40523} kind: Pod name: switch-graph-40523-6b6759674-ksx98 namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: 3013f16b-01cc-436e-be2a-691c507712f6 kind: Event lastTimestamp: "2026-07-02T13:55:02Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" metadata: creationTimestamp: "2026-07-02T13:55:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:02Z" name: switch-graph-40523-6b6759674-ksx98.18be7d82a3fba627 namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: 60d88e24-8d42-4b47-a1af-f7b546833f17 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-40523} kind: Pod name: switch-graph-40523-6b6759674-ksx98 namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: 3013f16b-01cc-436e-be2a-691c507712f6 kind: Event lastTimestamp: "2026-07-02T13:55:05Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" in 2.718s (2.718s including waiting). Image size: 218068124 bytes.' metadata: creationTimestamp: "2026-07-02T13:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:05Z" name: switch-graph-40523-6b6759674-ksx98.18be7d8346076f89 namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: bb0be74d-128b-4ed9-876f-7a3afdeb4bc9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-40523} kind: Pod name: switch-graph-40523-6b6759674-ksx98 namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: 3013f16b-01cc-436e-be2a-691c507712f6 kind: Event lastTimestamp: "2026-07-02T13:55:05Z" message: 'Created container: switch-graph-40523' metadata: creationTimestamp: "2026-07-02T13:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:05Z" name: switch-graph-40523-6b6759674-ksx98.18be7d83491269a2 namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 6304d0f6-af4c-474b-ae86-36177da214ae reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-40523} kind: Pod name: switch-graph-40523-6b6759674-ksx98 namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: 3013f16b-01cc-436e-be2a-691c507712f6 kind: Event lastTimestamp: "2026-07-02T13:55:05Z" message: Started container switch-graph-40523 metadata: creationTimestamp: "2026-07-02T13:55:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:05Z" name: switch-graph-40523-6b6759674-ksx98.18be7d8349738c67 namespace: kserve-ci-e2e-test resourceVersion: "15358" uid: 6167fd8e-6d7e-4dff-8969-ca485794dfd7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-40523} kind: Pod name: switch-graph-40523-6b6759674-ksx98 namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: 3013f16b-01cc-436e-be2a-691c507712f6 kind: Event lastTimestamp: "2026-07-02T13:55:15Z" message: Stopping container switch-graph-40523 metadata: creationTimestamp: "2026-07-02T13:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:15Z" name: switch-graph-40523-6b6759674-ksx98.18be7d85af1c4bd1 namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: c3ba6c9e-3634-4f23-8783-419503a8228a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:55:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-40523} kind: Pod name: switch-graph-40523-6b6759674-ksx98 namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: 3013f16b-01cc-436e-be2a-691c507712f6 kind: Event lastTimestamp: "2026-07-02T13:55:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:55:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:55:42Z" name: switch-graph-40523-6b6759674-ksx98.18be7d860a0a7e76 namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: 811769e9-cc10-4c32-a0be-f93f722d1ad4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-40523-6b6759674 namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 6fdb34ad-39b1-4622-b7fe-0cb69f6d9bd4 kind: Event lastTimestamp: "2026-07-02T13:55:01Z" message: 'Created pod: switch-graph-40523-6b6759674-ksx98' metadata: creationTimestamp: "2026-07-02T13:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:01Z" name: switch-graph-40523-6b6759674.18be7d8264d1be4d namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 52f4c583-aded-4e3c-9170-cfea7282e5b8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-40523 namespace: kserve-ci-e2e-test resourceVersion: "15304" uid: 994e9783-dd24-46a9-b026-0f61cf9cea05 kind: Event lastTimestamp: "2026-07-02T13:55:01Z" message: Scaled up replica set switch-graph-40523-6b6759674 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:55:01Z" name: switch-graph-40523.18be7d8263ad29f8 namespace: kserve-ci-e2e-test resourceVersion: "15307" uid: bae181e0-fb52-4018-b5f9-f28ea38fca08 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:55:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-40523 namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: f0fdfa19-4a6a-4b09-824f-dc056cb98588 kind: Event lastTimestamp: "2026-07-02T13:55:01Z" message: InferenceGraph [switch-graph-40523] is Ready metadata: creationTimestamp: "2026-07-02T13:55:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:55:01Z" name: switch-graph-40523.18be7d8269bc8400 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 8e90a94f-bd3e-466a-8cb9-1eff394fc136 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:56:15.883036Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-68692-78776d6ccd-fsth8 namespace: kserve-ci-e2e-test resourceVersion: "16334" uid: b7e56553-8dc5-47e1-9355-cbf353a81268 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-68692-78776d6ccd-fsth8 to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T13:56:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:56:15Z" name: switch-graph-68692-78776d6ccd-fsth8.18be7d93b7d370ae namespace: kserve-ci-e2e-test resourceVersion: "16339" uid: eb269e37-e6cd-406f-a19c-673bff62c48f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:16Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-68692-78776d6ccd-fsth8 namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: b7e56553-8dc5-47e1-9355-cbf353a81268 kind: Event lastTimestamp: "2026-07-02T13:56:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-68692-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:16Z" name: switch-graph-68692-78776d6ccd-fsth8.18be7d93c0761f37 namespace: kserve-ci-e2e-test resourceVersion: "16354" uid: a480c7de-8498-49c0-8383-e2fb83c71a61 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:16Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-68692-78776d6ccd-fsth8 namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: b7e56553-8dc5-47e1-9355-cbf353a81268 kind: Event lastTimestamp: "2026-07-02T13:56:16Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:56:16Z" name: switch-graph-68692-78776d6ccd-fsth8.18be7d93f53b88ea namespace: kserve-ci-e2e-test resourceVersion: "16364" uid: eae32e2a-765f-476f-9a73-379adea56bb5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-68692} kind: Pod name: switch-graph-68692-78776d6ccd-fsth8 namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: b7e56553-8dc5-47e1-9355-cbf353a81268 kind: Event lastTimestamp: "2026-07-02T13:56:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" already present on machine metadata: creationTimestamp: "2026-07-02T13:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:16Z" name: switch-graph-68692-78776d6ccd-fsth8.18be7d93f6a8523e namespace: kserve-ci-e2e-test resourceVersion: "16366" uid: 0c85e8e7-8f39-4d0b-87ea-af11b59f6e4b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-68692} kind: Pod name: switch-graph-68692-78776d6ccd-fsth8 namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: b7e56553-8dc5-47e1-9355-cbf353a81268 kind: Event lastTimestamp: "2026-07-02T13:56:16Z" message: 'Created container: switch-graph-68692' metadata: creationTimestamp: "2026-07-02T13:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:16Z" name: switch-graph-68692-78776d6ccd-fsth8.18be7d93fa444ce9 namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: 25dec7da-118c-4cde-947c-6faa019f0876 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-68692} kind: Pod name: switch-graph-68692-78776d6ccd-fsth8 namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: b7e56553-8dc5-47e1-9355-cbf353a81268 kind: Event lastTimestamp: "2026-07-02T13:56:17Z" message: Started container switch-graph-68692 metadata: creationTimestamp: "2026-07-02T13:56:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:56:17Z" name: switch-graph-68692-78776d6ccd-fsth8.18be7d93faa73b48 namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: 7e6f96a6-9426-4a2e-80aa-e3960610955c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:04:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-68692} kind: Pod name: switch-graph-68692-78776d6ccd-fsth8 namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: b7e56553-8dc5-47e1-9355-cbf353a81268 kind: Event lastTimestamp: "2026-07-02T14:04:30Z" message: Stopping container switch-graph-68692 metadata: creationTimestamp: "2026-07-02T14:04:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:30Z" name: switch-graph-68692-78776d6ccd-fsth8.18be7e06db4f8e6a namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: e87f4e29-093f-455e-9ea7-9a6ce699a376 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:04:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-68692} kind: Pod name: switch-graph-68692-78776d6ccd-fsth8 namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: b7e56553-8dc5-47e1-9355-cbf353a81268 kind: Event lastTimestamp: "2026-07-02T14:04:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:04:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:04:58Z" name: switch-graph-68692-78776d6ccd-fsth8.18be7e0791daeeda namespace: kserve-ci-e2e-test resourceVersion: "20203" uid: 49555dfd-95b3-473b-8384-9cf5f403dff7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-68692-78776d6ccd namespace: kserve-ci-e2e-test resourceVersion: "16331" uid: d00c169a-1b1f-406b-a040-3a325d8c28c5 kind: Event lastTimestamp: "2026-07-02T13:56:15Z" message: 'Created pod: switch-graph-68692-78776d6ccd-fsth8' metadata: creationTimestamp: "2026-07-02T13:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:15Z" name: switch-graph-68692-78776d6ccd.18be7d93b719e742 namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: a450bb0f-fa48-4b42-a03b-af9e4b4cc9e6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-68692 namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: 8efa0e14-7306-4a82-96f2-6a65621985d9 kind: Event lastTimestamp: "2026-07-02T13:56:15Z" message: Scaled up replica set switch-graph-68692-78776d6ccd from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:15Z" name: switch-graph-68692.18be7d93b655f7dc namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: 2c9a7dcc-6431-483a-8179-62382449423e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-68692 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 7bf5c190-5f32-4d66-abf0-6720da598748 kind: Event lastTimestamp: "2026-07-02T13:56:15Z" message: InferenceGraph [switch-graph-68692] is Ready metadata: creationTimestamp: "2026-07-02T13:56:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:56:15Z" name: switch-graph-68692.18be7d93bc1c79db namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: 7975615b-9eee-408d-8dd3-6464265e07fc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-68692 namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 13a7edc8-9e19-4104-828d-830b0090a47e kind: Event lastTimestamp: "2026-07-02T13:56:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:56:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:30Z" name: switch-graph-68692.18be7d9734cd7068 namespace: kserve-ci-e2e-test resourceVersion: "16493" uid: 605368c0-ee8c-4baa-8b21-e8b0fe6612be reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-68692 namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 13a7edc8-9e19-4104-828d-830b0090a47e kind: Event lastTimestamp: "2026-07-02T13:56:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:56:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:30Z" name: switch-graph-68692.18be7d97351a7be5 namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: 79b83600-d871-4d4e-8beb-e16586c1da70 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-68692 namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 13a7edc8-9e19-4104-828d-830b0090a47e kind: Event lastTimestamp: "2026-07-02T13:56:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:45Z" name: switch-graph-68692.18be7d9ab3c54814 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: 9cad879f-3180-4aff-aa18-eda89ef726cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:56:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-68692 namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 13a7edc8-9e19-4104-828d-830b0090a47e kind: Event lastTimestamp: "2026-07-02T13:56:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:56:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:56:45Z" name: switch-graph-68692.18be7d9ab42dbbf4 namespace: kserve-ci-e2e-test resourceVersion: "16613" uid: f11e1cdf-18e3-4df7-abc5-82fbaa6774a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T14:16:51.937151Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8d2ee-545b848957-x45gp namespace: kserve-ci-e2e-test resourceVersion: "26809" uid: f159dcf8-4505-42e0-86ce-22e768bdf8a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-8d2ee-545b848957-x45gp to ip-10-0-138-214.ec2.internal metadata: creationTimestamp: "2026-07-02T14:16:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T14:16:51Z" name: switch-graph-8d2ee-545b848957-x45gp.18be7eb38264689f namespace: kserve-ci-e2e-test resourceVersion: "26814" uid: 906b9b87-c2f8-4e02-bda7-f8a707291800 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-855788fc74-7k8mn source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:52Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8d2ee-545b848957-x45gp namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: f159dcf8-4505-42e0-86ce-22e768bdf8a6 kind: Event lastTimestamp: "2026-07-02T14:16:52Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-8d2ee-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T14:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:52Z" name: switch-graph-8d2ee-545b848957-x45gp.18be7eb391fc08f4 namespace: kserve-ci-e2e-test resourceVersion: "26827" uid: 647ddffd-9cef-4b61-b782-44c6a6f27139 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:52Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8d2ee-545b848957-x45gp namespace: kserve-ci-e2e-test resourceVersion: "26817" uid: f159dcf8-4505-42e0-86ce-22e768bdf8a6 kind: Event lastTimestamp: "2026-07-02T14:16:52Z" message: Add eth0 [10.134.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T14:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T14:16:52Z" name: switch-graph-8d2ee-545b848957-x45gp.18be7eb3bf9100cb namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 6faf7ba0-600c-47ca-9e24-67410b366ce4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8d2ee} kind: Pod name: switch-graph-8d2ee-545b848957-x45gp namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: f159dcf8-4505-42e0-86ce-22e768bdf8a6 kind: Event lastTimestamp: "2026-07-02T14:16:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:b87693b966d6607544232e73341be0a7df8bc57efa51891457e864fb9d5cc954" already present on machine metadata: creationTimestamp: "2026-07-02T14:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:52Z" name: switch-graph-8d2ee-545b848957-x45gp.18be7eb3c0ecaf18 namespace: kserve-ci-e2e-test resourceVersion: "26846" uid: f849df29-8b03-4cc4-8066-6b8a33d53003 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8d2ee} kind: Pod name: switch-graph-8d2ee-545b848957-x45gp namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: f159dcf8-4505-42e0-86ce-22e768bdf8a6 kind: Event lastTimestamp: "2026-07-02T14:16:53Z" message: 'Created container: switch-graph-8d2ee' metadata: creationTimestamp: "2026-07-02T14:16:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:53Z" name: switch-graph-8d2ee-545b848957-x45gp.18be7eb3c46de3c2 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: e69fa728-c5d6-4f03-a21e-29e0a7006d64 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8d2ee} kind: Pod name: switch-graph-8d2ee-545b848957-x45gp namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: f159dcf8-4505-42e0-86ce-22e768bdf8a6 kind: Event lastTimestamp: "2026-07-02T14:16:53Z" message: Started container switch-graph-8d2ee metadata: creationTimestamp: "2026-07-02T14:16:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:16:53Z" name: switch-graph-8d2ee-545b848957-x45gp.18be7eb3c4d97b36 namespace: kserve-ci-e2e-test resourceVersion: "26849" uid: ef11425d-8b8b-4963-a8f9-0779884a8420 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:33:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8d2ee} kind: Pod name: switch-graph-8d2ee-545b848957-x45gp namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: f159dcf8-4505-42e0-86ce-22e768bdf8a6 kind: Event lastTimestamp: "2026-07-02T14:33:11Z" message: Stopping container switch-graph-8d2ee metadata: creationTimestamp: "2026-07-02T14:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:33:11Z" name: switch-graph-8d2ee-545b848957-x45gp.18be7f97902e99d2 namespace: kserve-ci-e2e-test resourceVersion: "33681" uid: e0ca38a1-d60d-4b95-9ee4-a7d43666ffb3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T14:33:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8d2ee} kind: Pod name: switch-graph-8d2ee-545b848957-x45gp namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: f159dcf8-4505-42e0-86ce-22e768bdf8a6 kind: Event lastTimestamp: "2026-07-02T14:33:39Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T14:33:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T14:33:39Z" name: switch-graph-8d2ee-545b848957-x45gp.18be7f98644b0450 namespace: kserve-ci-e2e-test resourceVersion: "33958" uid: 3f1a139a-e400-4073-b9ec-fab78a2475ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-214.ec2.internal source: component: kubelet host: ip-10-0-138-214.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-8d2ee-545b848957 namespace: kserve-ci-e2e-test resourceVersion: "26805" uid: 28001b01-432c-4f2f-ac21-61f0a2bd609f kind: Event lastTimestamp: "2026-07-02T14:16:51Z" message: 'Created pod: switch-graph-8d2ee-545b848957-x45gp' metadata: creationTimestamp: "2026-07-02T14:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:16:51Z" name: switch-graph-8d2ee-545b848957.18be7eb381c863c0 namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: 73af7eb8-3c38-457c-8b34-79d33b11105f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-8d2ee namespace: kserve-ci-e2e-test resourceVersion: "26804" uid: becb2a8f-d010-45cf-8811-eb4322de0eb1 kind: Event lastTimestamp: "2026-07-02T14:16:51Z" message: Scaled up replica set switch-graph-8d2ee-545b848957 from 0 to 1 metadata: creationTimestamp: "2026-07-02T14:16:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:16:51Z" name: switch-graph-8d2ee.18be7eb3805d5de4 namespace: kserve-ci-e2e-test resourceVersion: "26807" uid: 95a87046-2cd5-45d5-80f5-b9ddc3cbfbe3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T14:16:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-8d2ee namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: 6dacb567-86a2-497d-8ec2-8eeaffc94d65 kind: Event lastTimestamp: "2026-07-02T14:16:51Z" message: InferenceGraph [switch-graph-8d2ee] is Ready metadata: creationTimestamp: "2026-07-02T14:16:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T14:16:51Z" name: switch-graph-8d2ee.18be7eb3860205b2 namespace: kserve-ci-e2e-test resourceVersion: "26825" uid: a19f1fb2-6320-4db6-988a-d9f76e98fbc8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:17:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8d2ee namespace: kserve-ci-e2e-test resourceVersion: "26806" uid: 1c7e9e6c-3af4-4e10-8c9b-4a413c60322c kind: Event lastTimestamp: "2026-07-02T14:17:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:21Z" name: switch-graph-8d2ee.18be7eb6feffc6e9 namespace: kserve-ci-e2e-test resourceVersion: "27120" uid: 2785d3bb-6c67-4def-b867-5baa0a380d97 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T14:17:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-8d2ee namespace: kserve-ci-e2e-test resourceVersion: "26806" uid: 1c7e9e6c-3af4-4e10-8c9b-4a413c60322c kind: Event lastTimestamp: "2026-07-02T14:17:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T14:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T14:17:21Z" name: switch-graph-8d2ee.18be7eb6ff676c89 namespace: kserve-ci-e2e-test resourceVersion: "27121" uid: 2dba059c-802a-4ad8-9b47-816069e0fd17 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "33958"