--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:24:49.726760Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3c295-5b8f7d666d-97k6q namespace: kserve-ci-e2e-test resourceVersion: "23596" uid: 44e1e169-dfc2-490f-acb5-e37909e1b08f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-3c295-5b8f7d666d-97k6q to ip-10-0-128-182.ec2.internal metadata: creationTimestamp: "2026-09-09T20:24:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:24:49Z" name: ensemble-graph-3c295-5b8f7d666d-97k6q.18d3c0d30f7df51d namespace: kserve-ci-e2e-test resourceVersion: "23601" uid: f043ec5f-426a-4be2-90bf-6571e2d6d697 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:50Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-3c295-5b8f7d666d-97k6q namespace: kserve-ci-e2e-test resourceVersion: "23605" uid: 44e1e169-dfc2-490f-acb5-e37909e1b08f kind: Event lastTimestamp: "2026-09-09T20:24:50Z" message: Add eth0 [10.133.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:24:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:24:50Z" name: ensemble-graph-3c295-5b8f7d666d-97k6q.18d3c0d3318759f6 namespace: kserve-ci-e2e-test resourceVersion: "23620" uid: acc51c15-68f6-4262-84f8-9e6620baf04d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3c295} kind: Pod name: ensemble-graph-3c295-5b8f7d666d-97k6q namespace: kserve-ci-e2e-test resourceVersion: "23598" uid: 44e1e169-dfc2-490f-acb5-e37909e1b08f kind: Event lastTimestamp: "2026-09-09T20:24:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:1c50c377478387906c73b97d7029d36701f6ae847fc2e48c877fa8c0c5df33ad" already present on machine metadata: creationTimestamp: "2026-09-09T20:24:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:50Z" name: ensemble-graph-3c295-5b8f7d666d-97k6q.18d3c0d332c9d08d namespace: kserve-ci-e2e-test resourceVersion: "23622" uid: 2e1058b3-57d8-450b-97ec-cab987cc6438 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3c295} kind: Pod name: ensemble-graph-3c295-5b8f7d666d-97k6q namespace: kserve-ci-e2e-test resourceVersion: "23598" uid: 44e1e169-dfc2-490f-acb5-e37909e1b08f kind: Event lastTimestamp: "2026-09-09T20:24:50Z" message: 'Created container: ensemble-graph-3c295' metadata: creationTimestamp: "2026-09-09T20:24:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:50Z" name: ensemble-graph-3c295-5b8f7d666d-97k6q.18d3c0d3358f8ad7 namespace: kserve-ci-e2e-test resourceVersion: "23623" uid: 388264f6-f64e-410a-bf7f-55015ff7c3cb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3c295} kind: Pod name: ensemble-graph-3c295-5b8f7d666d-97k6q namespace: kserve-ci-e2e-test resourceVersion: "23598" uid: 44e1e169-dfc2-490f-acb5-e37909e1b08f kind: Event lastTimestamp: "2026-09-09T20:24:50Z" message: Started container ensemble-graph-3c295 metadata: creationTimestamp: "2026-09-09T20:24:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:50Z" name: ensemble-graph-3c295-5b8f7d666d-97k6q.18d3c0d336085e38 namespace: kserve-ci-e2e-test resourceVersion: "23624" uid: f3b13a5b-ee8e-4a94-a487-e415e2d78d9e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3c295} kind: Pod name: ensemble-graph-3c295-5b8f7d666d-97k6q namespace: kserve-ci-e2e-test resourceVersion: "23598" uid: 44e1e169-dfc2-490f-acb5-e37909e1b08f kind: Event lastTimestamp: "2026-09-09T20:33:04Z" message: Stopping container ensemble-graph-3c295 metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:04Z" name: ensemble-graph-3c295-5b8f7d666d-97k6q.18d3c14639bb8bc3 namespace: kserve-ci-e2e-test resourceVersion: "27418" uid: b9685aea-fa7a-4719-9353-56c9093ef6e3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:33:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-3c295} kind: Pod name: ensemble-graph-3c295-5b8f7d666d-97k6q namespace: kserve-ci-e2e-test resourceVersion: "23598" uid: 44e1e169-dfc2-490f-acb5-e37909e1b08f kind: Event lastTimestamp: "2026-09-09T20:33:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-09T20:33:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:32Z" name: ensemble-graph-3c295-5b8f7d666d-97k6q.18d3c146e34afc65 namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 4df5cf8c-acc1-4e3e-8a4c-b050aba7c93e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-3c295-5b8f7d666d namespace: kserve-ci-e2e-test resourceVersion: "23593" uid: 47ae7371-7290-491e-aee7-50abf9ec36cd kind: Event lastTimestamp: "2026-09-09T20:24:49Z" message: 'Created pod: ensemble-graph-3c295-5b8f7d666d-97k6q' metadata: creationTimestamp: "2026-09-09T20:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:49Z" name: ensemble-graph-3c295-5b8f7d666d.18d3c0d30f1551ad namespace: kserve-ci-e2e-test resourceVersion: "23597" uid: e9c6fbe1-d85d-4cc7-a1db-b81cac1ceb49 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-3c295 namespace: kserve-ci-e2e-test resourceVersion: "23591" uid: 2ea00bc4-845f-45c7-bb3f-27c98863a5a4 kind: Event lastTimestamp: "2026-09-09T20:24:49Z" message: Scaled up replica set ensemble-graph-3c295-5b8f7d666d from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:49Z" name: ensemble-graph-3c295.18d3c0d30e699769 namespace: kserve-ci-e2e-test resourceVersion: "23594" uid: ba0faa4b-0c22-41ee-ab34-edeeb586bfd7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-3c295 namespace: kserve-ci-e2e-test resourceVersion: "23609" uid: c30140a5-883a-4567-be20-a8da8fc16e57 kind: Event lastTimestamp: "2026-09-09T20:24:49Z" message: InferenceGraph [ensemble-graph-3c295] is Ready metadata: creationTimestamp: "2026-09-09T20:24:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:24:49Z" name: ensemble-graph-3c295.18d3c0d3140582e3 namespace: kserve-ci-e2e-test resourceVersion: "23611" uid: 5b70e797-b5e7-4fae-8294-cd0210892425 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:25:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-3c295 namespace: kserve-ci-e2e-test resourceVersion: "23592" uid: 0b7968cc-079c-4902-be2f-35c38ad832e7 kind: Event lastTimestamp: "2026-09-09T20:25:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:25:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:25:19Z" name: ensemble-graph-3c295.18d3c0d68d3f8e0c namespace: kserve-ci-e2e-test resourceVersion: "23864" uid: d5f9717b-cd2e-4f0a-946f-a72e8b35f482 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:25:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-3c295 namespace: kserve-ci-e2e-test resourceVersion: "23592" uid: 0b7968cc-079c-4902-be2f-35c38ad832e7 kind: Event lastTimestamp: "2026-09-09T20:25:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:25:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:25:19Z" name: ensemble-graph-3c295.18d3c0d68da187a8 namespace: kserve-ci-e2e-test resourceVersion: "23865" uid: 99de763e-f00b-40d6-a814-ae987d089cad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:23:39.448905Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e029b-9b5f5b85c-wng4z namespace: kserve-ci-e2e-test resourceVersion: "22622" uid: a919057a-febe-419d-959f-7cfbd3cf3300 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-e029b-9b5f5b85c-wng4z to ip-10-0-128-182.ec2.internal metadata: creationTimestamp: "2026-09-09T20:23:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:23:39Z" name: ensemble-graph-e029b-9b5f5b85c-wng4z.18d3c0c2b29b0b80 namespace: kserve-ci-e2e-test resourceVersion: "22631" uid: 881eb013-9b48-41d3-adcd-e0f38386462b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:39Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e029b-9b5f5b85c-wng4z namespace: kserve-ci-e2e-test resourceVersion: "22626" uid: a919057a-febe-419d-959f-7cfbd3cf3300 kind: Event lastTimestamp: "2026-09-09T20:23:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-e029b-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:39Z" name: ensemble-graph-e029b-9b5f5b85c-wng4z.18d3c0c2bbc93dd3 namespace: kserve-ci-e2e-test resourceVersion: "22645" uid: 78eed3fe-43e4-4fa6-b912-08ef5c9356ed reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:40Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e029b-9b5f5b85c-wng4z namespace: kserve-ci-e2e-test resourceVersion: "22634" uid: a919057a-febe-419d-959f-7cfbd3cf3300 kind: Event lastTimestamp: "2026-09-09T20:23:40Z" message: Add eth0 [10.133.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:23:40Z" name: ensemble-graph-e029b-9b5f5b85c-wng4z.18d3c0c2f8671f16 namespace: kserve-ci-e2e-test resourceVersion: "22661" uid: d46f3dd6-0ae0-46a8-aeec-3ecd308757cf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e029b} kind: Pod name: ensemble-graph-e029b-9b5f5b85c-wng4z namespace: kserve-ci-e2e-test resourceVersion: "22626" uid: a919057a-febe-419d-959f-7cfbd3cf3300 kind: Event lastTimestamp: "2026-09-09T20:23:40Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:1c50c377478387906c73b97d7029d36701f6ae847fc2e48c877fa8c0c5df33ad" already present on machine metadata: creationTimestamp: "2026-09-09T20:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:40Z" name: ensemble-graph-e029b-9b5f5b85c-wng4z.18d3c0c2f9aa952d namespace: kserve-ci-e2e-test resourceVersion: "22664" uid: 0f62ec18-2f6b-4376-9efc-208faa42f644 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e029b} kind: Pod name: ensemble-graph-e029b-9b5f5b85c-wng4z namespace: kserve-ci-e2e-test resourceVersion: "22626" uid: a919057a-febe-419d-959f-7cfbd3cf3300 kind: Event lastTimestamp: "2026-09-09T20:23:40Z" message: 'Created container: ensemble-graph-e029b' metadata: creationTimestamp: "2026-09-09T20:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:40Z" name: ensemble-graph-e029b-9b5f5b85c-wng4z.18d3c0c2fd004830 namespace: kserve-ci-e2e-test resourceVersion: "22666" uid: 879c72e2-f25c-4073-8d17-176c1a183477 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e029b} kind: Pod name: ensemble-graph-e029b-9b5f5b85c-wng4z namespace: kserve-ci-e2e-test resourceVersion: "22626" uid: a919057a-febe-419d-959f-7cfbd3cf3300 kind: Event lastTimestamp: "2026-09-09T20:23:40Z" message: Started container ensemble-graph-e029b metadata: creationTimestamp: "2026-09-09T20:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:40Z" name: ensemble-graph-e029b-9b5f5b85c-wng4z.18d3c0c2fd750418 namespace: kserve-ci-e2e-test resourceVersion: "22667" uid: 2b8ec25d-d3bd-420a-98ba-ac885ec56ec5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e029b} kind: Pod name: ensemble-graph-e029b-9b5f5b85c-wng4z namespace: kserve-ci-e2e-test resourceVersion: "22626" uid: a919057a-febe-419d-959f-7cfbd3cf3300 kind: Event lastTimestamp: "2026-09-09T20:23:49Z" message: Stopping container ensemble-graph-e029b metadata: creationTimestamp: "2026-09-09T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:49Z" name: ensemble-graph-e029b-9b5f5b85c-wng4z.18d3c0c50ad8e7c6 namespace: kserve-ci-e2e-test resourceVersion: "22756" uid: 85ef9fac-601e-4679-9b82-f58936dde435 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e029b} kind: Pod name: ensemble-graph-e029b-9b5f5b85c-wng4z namespace: kserve-ci-e2e-test resourceVersion: "22626" uid: a919057a-febe-419d-959f-7cfbd3cf3300 kind: Event lastTimestamp: "2026-09-09T20:24:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-09T20:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:16Z" name: ensemble-graph-e029b-9b5f5b85c-wng4z.18d3c0c5973c85e2 namespace: kserve-ci-e2e-test resourceVersion: "23118" uid: b177c3f9-a9bc-4a61-9147-669cef4782f5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-e029b-9b5f5b85c namespace: kserve-ci-e2e-test resourceVersion: "22618" uid: c6a77a56-4d79-4771-9e0d-616aceadc11d kind: Event lastTimestamp: "2026-09-09T20:23:39Z" message: 'Created pod: ensemble-graph-e029b-9b5f5b85c-wng4z' metadata: creationTimestamp: "2026-09-09T20:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:39Z" name: ensemble-graph-e029b-9b5f5b85c.18d3c0c2b2325c28 namespace: kserve-ci-e2e-test resourceVersion: "22625" uid: 4c9c6001-4b90-4f39-a279-065fbe64a052 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-e029b namespace: kserve-ci-e2e-test resourceVersion: "22616" uid: c31929d8-9f76-49c7-90d7-37e50ae339db kind: Event lastTimestamp: "2026-09-09T20:23:39Z" message: Scaled up replica set ensemble-graph-e029b-9b5f5b85c from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:39Z" name: ensemble-graph-e029b.18d3c0c2b17d3d2b namespace: kserve-ci-e2e-test resourceVersion: "22623" uid: f32a6445-36e4-45fb-a77e-0cc67d65067d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-e029b namespace: kserve-ci-e2e-test resourceVersion: "22637" uid: afe29907-befc-4147-b5ae-1b52138f8a78 kind: Event lastTimestamp: "2026-09-09T20:23:39Z" message: InferenceGraph [ensemble-graph-e029b] is Ready metadata: creationTimestamp: "2026-09-09T20:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:23:39Z" name: ensemble-graph-e029b.18d3c0c2b5d19ff0 namespace: kserve-ci-e2e-test resourceVersion: "22639" uid: a096a4a0-2afe-41a9-b261-331a7fabea41 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-e029b namespace: kserve-ci-e2e-test resourceVersion: "22611" uid: afe29907-befc-4147-b5ae-1b52138f8a78 kind: Event lastTimestamp: "2026-09-09T20:23:39Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-e029b": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-e029b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-09T20:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:23:39Z" name: ensemble-graph-e029b.18d3c0c2b840c775 namespace: kserve-ci-e2e-test resourceVersion: "22642" uid: cc82db0a-574c-4f65-9785-44784805e1d3 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-e029b namespace: kserve-ci-e2e-test resourceVersion: "22611" uid: afe29907-befc-4147-b5ae-1b52138f8a78 kind: Event lastTimestamp: "2026-09-09T20:23:39Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-e029b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-09T20:23:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:23:39Z" name: ensemble-graph-e029b.18d3c0c2b8411091 namespace: kserve-ci-e2e-test resourceVersion: "22643" uid: 4feb9142-6d59-4b8f-95d0-658d063eed01 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:14:10.781756Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c namespace: kserve-ci-e2e-test resourceVersion: "18148" uid: c407662e-cdc1-446c-9904-02069a00ab7a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2e942-predictor-7cb698b899-cbq9c to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:14:10Z" name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c.18d3c03e4b66f706 namespace: kserve-ci-e2e-test resourceVersion: "18152" uid: 2495ac5e-d472-476e-b728-3b359c1c9f0a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c namespace: kserve-ci-e2e-test resourceVersion: "18150" uid: c407662e-cdc1-446c-9904-02069a00ab7a kind: Event lastTimestamp: "2026-09-09T20:14:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-2e942-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:10Z" name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c.18d3c03e55053f66 namespace: kserve-ci-e2e-test resourceVersion: "18167" uid: c441582b-707d-4376-ab6e-e1c8b9c41fda reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c namespace: kserve-ci-e2e-test resourceVersion: "18158" uid: c407662e-cdc1-446c-9904-02069a00ab7a kind: Event lastTimestamp: "2026-09-09T20:14:11Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:14:11Z" name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c.18d3c03e912fd545 namespace: kserve-ci-e2e-test resourceVersion: "18192" uid: 6b9eff83-3a1c-4d6c-ae7b-994452eb7e24 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c namespace: kserve-ci-e2e-test resourceVersion: "18150" uid: c407662e-cdc1-446c-9904-02069a00ab7a kind: Event lastTimestamp: "2026-09-09T20:14:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:11Z" name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c.18d3c03e934ea97d namespace: kserve-ci-e2e-test resourceVersion: "18195" uid: 353ca194-09df-45b8-b21c-963f29acecc9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c namespace: kserve-ci-e2e-test resourceVersion: "18150" uid: c407662e-cdc1-446c-9904-02069a00ab7a kind: Event lastTimestamp: "2026-09-09T20:14:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:12Z" name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c.18d3c03e95a29ed2 namespace: kserve-ci-e2e-test resourceVersion: "18199" uid: a6ea3eaf-2123-4d2d-b43a-c654359f7541 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c namespace: kserve-ci-e2e-test resourceVersion: "18150" uid: c407662e-cdc1-446c-9904-02069a00ab7a kind: Event lastTimestamp: "2026-09-09T20:14:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:12Z" name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c.18d3c03e9612da29 namespace: kserve-ci-e2e-test resourceVersion: "18200" uid: b01730a7-a440-449f-9a0f-5a22a6e6ce83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c namespace: kserve-ci-e2e-test resourceVersion: "18150" uid: c407662e-cdc1-446c-9904-02069a00ab7a kind: Event lastTimestamp: "2026-09-09T20:14:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:12Z" name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c.18d3c03e961d30f5 namespace: kserve-ci-e2e-test resourceVersion: "18201" uid: a8fc1e5d-6cbd-486c-95c0-b4bdd8447042 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c namespace: kserve-ci-e2e-test resourceVersion: "18150" uid: c407662e-cdc1-446c-9904-02069a00ab7a kind: Event lastTimestamp: "2026-09-09T20:14:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:12Z" name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c.18d3c03e98b5320a namespace: kserve-ci-e2e-test resourceVersion: "18202" uid: 6deb66f2-df68-465c-b5aa-165eb5db12bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c namespace: kserve-ci-e2e-test resourceVersion: "18150" uid: c407662e-cdc1-446c-9904-02069a00ab7a kind: Event lastTimestamp: "2026-09-09T20:14:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:12Z" name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c.18d3c03e9928a042 namespace: kserve-ci-e2e-test resourceVersion: "18203" uid: 63841c0b-c275-431d-8ad1-7950de48e518 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c namespace: kserve-ci-e2e-test resourceVersion: "18150" uid: c407662e-cdc1-446c-9904-02069a00ab7a kind: Event lastTimestamp: "2026-09-09T20:14:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:48Z" name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c.18d3c03ec50a9d01 namespace: kserve-ci-e2e-test resourceVersion: "18571" uid: c9616ecc-00e5-4249-8f42-3ff7b76082d2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c namespace: kserve-ci-e2e-test resourceVersion: "18150" uid: c407662e-cdc1-446c-9904-02069a00ab7a kind: Event lastTimestamp: "2026-09-09T20:23:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:25Z" name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c.18d3c0bf65a458ac namespace: kserve-ci-e2e-test resourceVersion: "22398" uid: 1f826ecd-5da9-438c-9ed7-a2915ac7e61c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c namespace: kserve-ci-e2e-test resourceVersion: "18150" uid: c407662e-cdc1-446c-9904-02069a00ab7a kind: Event lastTimestamp: "2026-09-09T20:23:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:25Z" name: error-404-isvc-2e942-predictor-7cb698b899-cbq9c.18d3c0bf65a4de4b namespace: kserve-ci-e2e-test resourceVersion: "22400" uid: 9cccf129-5f9d-48be-ac97-2639e4f5849d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2e942-predictor-7cb698b899 namespace: kserve-ci-e2e-test resourceVersion: "18142" uid: 15db927a-b248-447f-ab7e-4035dede34c0 kind: Event lastTimestamp: "2026-09-09T20:14:10Z" message: 'Created pod: error-404-isvc-2e942-predictor-7cb698b899-cbq9c' metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:10Z" name: error-404-isvc-2e942-predictor-7cb698b899.18d3c03e4ad81c5f namespace: kserve-ci-e2e-test resourceVersion: "18149" uid: 7ed67470-ca96-45a8-a564-95deba7b5c03 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2e942-predictor namespace: kserve-ci-e2e-test resourceVersion: "18140" uid: a0eae0ab-cb8c-4577-9462-d94bbdd957df kind: Event lastTimestamp: "2026-09-09T20:14:10Z" message: Scaled up replica set error-404-isvc-2e942-predictor-7cb698b899 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:10Z" name: error-404-isvc-2e942-predictor.18d3c03e495d8661 namespace: kserve-ci-e2e-test resourceVersion: "18143" uid: dcaaa154-dcc0-4a4a-8e83-3195dfee60fe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:14:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2e942-predictor namespace: kserve-ci-e2e-test resourceVersion: "18141" uid: 2d7614d4-6980-4176-9f05-780af6fbdf3f kind: Event lastTimestamp: "2026-09-09T20:14:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:14:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:55Z" name: error-404-isvc-2e942-predictor.18d3c041c7e036be namespace: kserve-ci-e2e-test resourceVersion: "18624" uid: 962a28b9-2942-418e-85bd-8e3ba53959ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:14:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2e942-predictor namespace: kserve-ci-e2e-test resourceVersion: "18141" uid: 2d7614d4-6980-4176-9f05-780af6fbdf3f kind: Event lastTimestamp: "2026-09-09T20:14:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:14:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:55Z" name: error-404-isvc-2e942-predictor.18d3c041c81c7e75 namespace: kserve-ci-e2e-test resourceVersion: "18625" uid: 7ed4e210-cbfe-4349-bf3b-a121ab2be658 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:15:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2e942-predictor namespace: kserve-ci-e2e-test resourceVersion: "18375" uid: 2d7614d4-6980-4176-9f05-780af6fbdf3f kind: Event lastTimestamp: "2026-09-09T20:15:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:15:25Z" name: error-404-isvc-2e942-predictor.18d3c04c440b6c04 namespace: kserve-ci-e2e-test resourceVersion: "18901" uid: 1d2e056e-dd9f-481d-8d20-240496bd16da reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:15:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2e942-predictor namespace: kserve-ci-e2e-test resourceVersion: "18375" uid: 2d7614d4-6980-4176-9f05-780af6fbdf3f kind: Event lastTimestamp: "2026-09-09T20:15:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:15:25Z" name: error-404-isvc-2e942-predictor.18d3c04c4471e364 namespace: kserve-ci-e2e-test resourceVersion: "18902" uid: da2c41c0-98d6-4b53-bfaf-0d6de2082415 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2e942 namespace: kserve-ci-e2e-test resourceVersion: "18128" uid: 6cf8d52d-1a64-4170-b1dd-c529d337fbc7 kind: Event lastTimestamp: "2026-09-09T20:14:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:14:59Z" name: error-404-isvc-2e942.18d3c03e4965cb6b namespace: kserve-ci-e2e-test resourceVersion: "18663" uid: 5cd63ce2-4e25-4e9f-97ca-e90b5f798ad5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2e942 namespace: kserve-ci-e2e-test resourceVersion: "18656" uid: 6cf8d52d-1a64-4170-b1dd-c529d337fbc7 kind: Event lastTimestamp: "2026-09-09T20:14:58Z" message: InferenceService [error-404-isvc-2e942] is Ready metadata: creationTimestamp: "2026-09-09T20:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:14:58Z" name: error-404-isvc-2e942.18d3c04980ba67e3 namespace: kserve-ci-e2e-test resourceVersion: "18657" uid: 40f2d1e8-9b89-4f52-a2ca-796408c2db2a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:24:45.750381Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw namespace: kserve-ci-e2e-test resourceVersion: "23505" uid: af9a3567-7b10-43c0-94ac-13a1e6799579 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:24:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:24:45Z" name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw.18d3c0d2227b4b4d namespace: kserve-ci-e2e-test resourceVersion: "23511" uid: 0deba018-cef2-4ee6-9161-492d82eef049 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw namespace: kserve-ci-e2e-test resourceVersion: "23518" uid: af9a3567-7b10-43c0-94ac-13a1e6799579 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:24:46Z" name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw.18d3c0d2445603f6 namespace: kserve-ci-e2e-test resourceVersion: "23534" uid: 6ff9e57b-cce2-48f1-8a67-08b6b43746c8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw namespace: kserve-ci-e2e-test resourceVersion: "23508" uid: af9a3567-7b10-43c0-94ac-13a1e6799579 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:46Z" name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw.18d3c0d245af2660 namespace: kserve-ci-e2e-test resourceVersion: "23536" uid: 4b8e64be-51a2-46ed-bfcf-1505bf8d01c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw namespace: kserve-ci-e2e-test resourceVersion: "23508" uid: af9a3567-7b10-43c0-94ac-13a1e6799579 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:46Z" name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw.18d3c0d24825b27e namespace: kserve-ci-e2e-test resourceVersion: "23539" uid: 9c07fcb5-cc1f-47df-bd6f-b805f5f1b070 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw namespace: kserve-ci-e2e-test resourceVersion: "23508" uid: af9a3567-7b10-43c0-94ac-13a1e6799579 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:46Z" name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw.18d3c0d2489346b4 namespace: kserve-ci-e2e-test resourceVersion: "23540" uid: 5996eb24-0963-42d8-92d8-3d08d77804b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw namespace: kserve-ci-e2e-test resourceVersion: "23508" uid: af9a3567-7b10-43c0-94ac-13a1e6799579 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:46Z" name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw.18d3c0d2489ba31a namespace: kserve-ci-e2e-test resourceVersion: "23541" uid: 0659ddf6-bfb8-4728-b78a-447cea982e7f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw namespace: kserve-ci-e2e-test resourceVersion: "23508" uid: af9a3567-7b10-43c0-94ac-13a1e6799579 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:46Z" name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw.18d3c0d24b30102c namespace: kserve-ci-e2e-test resourceVersion: "23542" uid: b617bf6a-a350-42a4-9744-4cd86a5cc59e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw namespace: kserve-ci-e2e-test resourceVersion: "23508" uid: af9a3567-7b10-43c0-94ac-13a1e6799579 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:46Z" name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw.18d3c0d24ba1b3d9 namespace: kserve-ci-e2e-test resourceVersion: "23543" uid: 692b7680-72f4-46b0-baf9-28e687cd8f06 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-09T20:24:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw namespace: kserve-ci-e2e-test resourceVersion: "23508" uid: af9a3567-7b10-43c0-94ac-13a1e6799579 kind: Event lastTimestamp: "2026-09-09T20:34:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:24:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:03Z" name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw.18d3c0d2882911d4 namespace: kserve-ci-e2e-test resourceVersion: "28137" uid: c7c052b4-c3dd-4d2d-8ea6-59815a7e16f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw namespace: kserve-ci-e2e-test resourceVersion: "23508" uid: af9a3567-7b10-43c0-94ac-13a1e6799579 kind: Event lastTimestamp: "2026-09-09T20:34:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:00Z" name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw.18d3c153476ccae4 namespace: kserve-ci-e2e-test resourceVersion: "28030" uid: 36950ab0-a303-47c2-b157-dd56e472bc71 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw namespace: kserve-ci-e2e-test resourceVersion: "23508" uid: af9a3567-7b10-43c0-94ac-13a1e6799579 kind: Event lastTimestamp: "2026-09-09T20:34:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:00Z" name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw.18d3c153476d2f44 namespace: kserve-ci-e2e-test resourceVersion: "28031" uid: d799dc43-5cef-490d-a663-dbf8ca493664 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw namespace: kserve-ci-e2e-test resourceVersion: "23508" uid: af9a3567-7b10-43c0-94ac-13a1e6799579 kind: Event lastTimestamp: "2026-09-09T20:34:03Z" message: 'Readiness probe failed: Get "https://10.132.0.30:8643/healthz": dial tcp 10.132.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:03Z" name: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw.18d3c153fc93cab6 namespace: kserve-ci-e2e-test resourceVersion: "28134" uid: 1035ac51-ce72-4da1-8952-2b6a8bab1ce8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3a010-predictor-65b6cbcbcc namespace: kserve-ci-e2e-test resourceVersion: "23498" uid: 31abca38-4c4c-4cbc-b199-7324c7eefd72 kind: Event lastTimestamp: "2026-09-09T20:24:45Z" message: 'Created pod: error-404-isvc-3a010-predictor-65b6cbcbcc-gt8mw' metadata: creationTimestamp: "2026-09-09T20:24:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:45Z" name: error-404-isvc-3a010-predictor-65b6cbcbcc.18d3c0d2220573c8 namespace: kserve-ci-e2e-test resourceVersion: "23507" uid: 436864a2-fef2-4072-892c-437909a1618e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3a010-predictor namespace: kserve-ci-e2e-test resourceVersion: "23496" uid: 44038ab8-a574-4588-9a24-e2feaec3f5cc kind: Event lastTimestamp: "2026-09-09T20:24:45Z" message: Scaled up replica set error-404-isvc-3a010-predictor-65b6cbcbcc from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:24:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:45Z" name: error-404-isvc-3a010-predictor.18d3c0d2202ee4c8 namespace: kserve-ci-e2e-test resourceVersion: "23499" uid: dcd6c995-49d5-4579-845e-adff4d027a00 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:25:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a010-predictor namespace: kserve-ci-e2e-test resourceVersion: "23497" uid: 5972614f-90e5-46f9-82e0-3c5c72f3cd4b kind: Event lastTimestamp: "2026-09-09T20:26:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:26:00Z" name: error-404-isvc-3a010-predictor.18d3c0d59ea69f9e namespace: kserve-ci-e2e-test resourceVersion: "24505" uid: 9db8dd71-4c23-44dc-8c5e-ce4ce5117f7b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:25:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a010-predictor namespace: kserve-ci-e2e-test resourceVersion: "23497" uid: 5972614f-90e5-46f9-82e0-3c5c72f3cd4b kind: Event lastTimestamp: "2026-09-09T20:26:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:26:00Z" name: error-404-isvc-3a010-predictor.18d3c0d59f00cf00 namespace: kserve-ci-e2e-test resourceVersion: "24506" uid: 3dc616d6-a5b8-4237-8a45-fb810befe40d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:24:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3a010 namespace: kserve-ci-e2e-test resourceVersion: "23484" uid: afb2c82c-2df3-49f0-9b46-e4cbc45463be kind: Event lastTimestamp: "2026-09-09T20:25:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:24:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:25:33Z" name: error-404-isvc-3a010.18d3c0d22036a646 namespace: kserve-ci-e2e-test resourceVersion: "23983" uid: c25cd4b8-6b89-4fde-bd2d-bdeaac0427c7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:25:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3a010 namespace: kserve-ci-e2e-test resourceVersion: "23978" uid: afb2c82c-2df3-49f0-9b46-e4cbc45463be kind: Event lastTimestamp: "2026-09-09T20:25:33Z" message: InferenceService [error-404-isvc-3a010] is Ready metadata: creationTimestamp: "2026-09-09T20:25:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:25:33Z" name: error-404-isvc-3a010.18d3c0dd44294f56 namespace: kserve-ci-e2e-test resourceVersion: "23979" uid: 0593f102-031c-4194-9872-273f1ef79c3a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:11:50.774195Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3b763-predictor-786696fb7c-k98n4 namespace: kserve-ci-e2e-test resourceVersion: "16504" uid: a3a1cff0-78d5-4f5f-ace4-9fd633444128 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3b763-predictor-786696fb7c-k98n4 to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:11:50Z" name: error-404-isvc-3b763-predictor-786696fb7c-k98n4.18d3c01db24d2ab1 namespace: kserve-ci-e2e-test resourceVersion: "16509" uid: d4fff60a-d6f4-4184-b367-e6e563115ba9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3b763-predictor-786696fb7c-k98n4 namespace: kserve-ci-e2e-test resourceVersion: "16514" uid: a3a1cff0-78d5-4f5f-ace4-9fd633444128 kind: Event lastTimestamp: "2026-09-09T20:11:51Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:11:51Z" name: error-404-isvc-3b763-predictor-786696fb7c-k98n4.18d3c01dd4211dfb namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: 71f2156c-3e2a-47dd-8ff4-4c115de371d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3b763-predictor-786696fb7c-k98n4 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: a3a1cff0-78d5-4f5f-ace4-9fd633444128 kind: Event lastTimestamp: "2026-09-09T20:11:51Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1907" metadata: creationTimestamp: "2026-09-09T20:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:51Z" name: error-404-isvc-3b763-predictor-786696fb7c-k98n4.18d3c01dd5ce2488 namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: 4e45dd65-0cee-4df1-a6f1-1a590f0784f8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3b763-predictor-786696fb7c-k98n4 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: a3a1cff0-78d5-4f5f-ace4-9fd633444128 kind: Event lastTimestamp: "2026-09-09T20:12:07Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1907" in 16.273s (16.273s including waiting). Image size: 1355573313 bytes.' metadata: creationTimestamp: "2026-09-09T20:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:07Z" name: error-404-isvc-3b763-predictor-786696fb7c-k98n4.18d3c0219fc428cb namespace: kserve-ci-e2e-test resourceVersion: "16744" uid: 56d37d7f-f5bb-407d-85bb-232bc4c0b9ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3b763-predictor-786696fb7c-k98n4 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: a3a1cff0-78d5-4f5f-ace4-9fd633444128 kind: Event lastTimestamp: "2026-09-09T20:12:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:07Z" name: error-404-isvc-3b763-predictor-786696fb7c-k98n4.18d3c021a390728d namespace: kserve-ci-e2e-test resourceVersion: "16747" uid: b504e61a-603f-418e-b55d-3695b8673c7c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3b763-predictor-786696fb7c-k98n4 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: a3a1cff0-78d5-4f5f-ace4-9fd633444128 kind: Event lastTimestamp: "2026-09-09T20:12:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:07Z" name: error-404-isvc-3b763-predictor-786696fb7c-k98n4.18d3c021a4100d10 namespace: kserve-ci-e2e-test resourceVersion: "16749" uid: 23c7ad87-d7ab-4f81-8ea9-3198399e3597 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3b763-predictor-786696fb7c-k98n4 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: a3a1cff0-78d5-4f5f-ace4-9fd633444128 kind: Event lastTimestamp: "2026-09-09T20:12:07Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-09T20:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:07Z" name: error-404-isvc-3b763-predictor-786696fb7c-k98n4.18d3c021a41e6c48 namespace: kserve-ci-e2e-test resourceVersion: "16752" uid: 1a905e45-986e-4ccf-aa32-fb8621bcf242 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3b763-predictor-786696fb7c-k98n4 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: a3a1cff0-78d5-4f5f-ace4-9fd633444128 kind: Event lastTimestamp: "2026-09-09T20:12:10Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.622s (2.622s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-09T20:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:10Z" name: error-404-isvc-3b763-predictor-786696fb7c-k98n4.18d3c022406f314b namespace: kserve-ci-e2e-test resourceVersion: "16775" uid: 7dce03c2-701c-4b89-baac-01332021d4e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3b763-predictor-786696fb7c-k98n4 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: a3a1cff0-78d5-4f5f-ace4-9fd633444128 kind: Event lastTimestamp: "2026-09-09T20:12:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:10Z" name: error-404-isvc-3b763-predictor-786696fb7c-k98n4.18d3c02243bd2e93 namespace: kserve-ci-e2e-test resourceVersion: "16778" uid: b5d6539e-dd43-43b7-9515-787c6bce21d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3b763-predictor-786696fb7c-k98n4 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: a3a1cff0-78d5-4f5f-ace4-9fd633444128 kind: Event lastTimestamp: "2026-09-09T20:12:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:10Z" name: error-404-isvc-3b763-predictor-786696fb7c-k98n4.18d3c02244489675 namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: 5817546d-34b4-47a9-a769-83a514cdc075 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:12:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3b763-predictor-786696fb7c-k98n4 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: a3a1cff0-78d5-4f5f-ace4-9fd633444128 kind: Event lastTimestamp: "2026-09-09T20:12:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:12:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:48Z" name: error-404-isvc-3b763-predictor-786696fb7c-k98n4.18d3c022b8db3f50 namespace: kserve-ci-e2e-test resourceVersion: "17117" uid: 4bbefb1a-f997-42ae-b218-0a022d40cde0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3b763-predictor-786696fb7c-k98n4 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: a3a1cff0-78d5-4f5f-ace4-9fd633444128 kind: Event lastTimestamp: "2026-09-09T20:13:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:24Z" name: error-404-isvc-3b763-predictor-786696fb7c-k98n4.18d3c03397eb08aa namespace: kserve-ci-e2e-test resourceVersion: "17575" uid: b66e67a1-c6ce-41a7-a3d7-9c57e4e3bee0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3b763-predictor-786696fb7c-k98n4 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: a3a1cff0-78d5-4f5f-ace4-9fd633444128 kind: Event lastTimestamp: "2026-09-09T20:13:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:24Z" name: error-404-isvc-3b763-predictor-786696fb7c-k98n4.18d3c03397eb5eae namespace: kserve-ci-e2e-test resourceVersion: "17577" uid: 50e6d6ab-0fc6-49a7-b6e9-ea9dda7cb0ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3b763-predictor-786696fb7c namespace: kserve-ci-e2e-test resourceVersion: "16498" uid: adb7ad94-1a08-41a7-8a27-3f525b74b88b kind: Event lastTimestamp: "2026-09-09T20:11:50Z" message: 'Created pod: error-404-isvc-3b763-predictor-786696fb7c-k98n4' metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:11:50Z" name: error-404-isvc-3b763-predictor-786696fb7c.18d3c01db1ce1d76 namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 8048d9f0-de95-4b8d-80cb-8f5185edf0ec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3b763-predictor namespace: kserve-ci-e2e-test resourceVersion: "16496" uid: 5e044427-3107-414a-80f4-0fa8f7eb8f79 kind: Event lastTimestamp: "2026-09-09T20:11:50Z" message: Scaled up replica set error-404-isvc-3b763-predictor-786696fb7c from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:11:50Z" name: error-404-isvc-3b763-predictor.18d3c01db0035d6d namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 8ac97ea2-0be3-4487-8258-83c90f00b114 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3b763-predictor namespace: kserve-ci-e2e-test resourceVersion: "16497" uid: e182606e-3b12-4833-8a70-96ea15bb6651 kind: Event lastTimestamp: "2026-09-09T20:12:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:12:35Z" name: error-404-isvc-3b763-predictor.18d3c0212e858a28 namespace: kserve-ci-e2e-test resourceVersion: "17021" uid: e2b26a39-a931-40e7-9d32-951ce170b4b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3b763-predictor namespace: kserve-ci-e2e-test resourceVersion: "16497" uid: e182606e-3b12-4833-8a70-96ea15bb6651 kind: Event lastTimestamp: "2026-09-09T20:12:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:12:35Z" name: error-404-isvc-3b763-predictor.18d3c0212ec3c73b namespace: kserve-ci-e2e-test resourceVersion: "17022" uid: 63acd0a8-1fed-407a-9e0b-5980965226d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:12:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3b763-predictor namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: e182606e-3b12-4833-8a70-96ea15bb6651 kind: Event lastTimestamp: "2026-09-09T20:13:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:12:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:20Z" name: error-404-isvc-3b763-predictor.18d3c02bababb0e0 namespace: kserve-ci-e2e-test resourceVersion: "17438" uid: 9ebf84a0-c250-45d2-b842-39d96cf98e3e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:12:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3b763-predictor namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: e182606e-3b12-4833-8a70-96ea15bb6651 kind: Event lastTimestamp: "2026-09-09T20:13:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:12:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:20Z" name: error-404-isvc-3b763-predictor.18d3c02bac0f1493 namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: 8f25ef4f-9136-404a-8c7c-5a49a3e6d0d5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3b763 namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: 64d3cde8-8121-4dcd-8681-0cbc1d5149ad kind: Event lastTimestamp: "2026-09-09T20:12:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:12:58Z" name: error-404-isvc-3b763.18d3c01db00f89bb namespace: kserve-ci-e2e-test resourceVersion: "17216" uid: cbde8c96-1e5b-4e96-b386-6da98c74a4c8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3b763 namespace: kserve-ci-e2e-test resourceVersion: "17214" uid: 64d3cde8-8121-4dcd-8681-0cbc1d5149ad kind: Event lastTimestamp: "2026-09-09T20:12:58Z" message: InferenceService [error-404-isvc-3b763] is Ready metadata: creationTimestamp: "2026-09-09T20:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:12:58Z" name: error-404-isvc-3b763.18d3c02d73fdeba9 namespace: kserve-ci-e2e-test resourceVersion: "17215" uid: 856dfdee-ce07-4721-95e3-7bf866a7ed7e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:23:49.837453Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3c295-predictor-6f998d6587-44pj7 namespace: kserve-ci-e2e-test resourceVersion: "22832" uid: 2f8c5e33-99fa-4e19-9286-eb3f5ec93e20 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3c295-predictor-6f998d6587-44pj7 to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:23:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:23:49Z" name: error-404-isvc-3c295-predictor-6f998d6587-44pj7.18d3c0c51dcfbbe9 namespace: kserve-ci-e2e-test resourceVersion: "22837" uid: c6d44b7d-5c7b-4347-b354-e8759e449da1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3c295-predictor-6f998d6587-44pj7 namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: 2f8c5e33-99fa-4e19-9286-eb3f5ec93e20 kind: Event lastTimestamp: "2026-09-09T20:23:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-3c295-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:50Z" name: error-404-isvc-3c295-predictor-6f998d6587-44pj7.18d3c0c5284fe440 namespace: kserve-ci-e2e-test resourceVersion: "22852" uid: 232a1fc6-7ddb-45c7-938f-0df95ad4a63e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3c295-predictor-6f998d6587-44pj7 namespace: kserve-ci-e2e-test resourceVersion: "22844" uid: 2f8c5e33-99fa-4e19-9286-eb3f5ec93e20 kind: Event lastTimestamp: "2026-09-09T20:23:51Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:23:51Z" name: error-404-isvc-3c295-predictor-6f998d6587-44pj7.18d3c0c563aa8930 namespace: kserve-ci-e2e-test resourceVersion: "22870" uid: f88b495f-19d8-4ae9-9d0a-7e62a341b705 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c295-predictor-6f998d6587-44pj7 namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: 2f8c5e33-99fa-4e19-9286-eb3f5ec93e20 kind: Event lastTimestamp: "2026-09-09T20:23:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:51Z" name: error-404-isvc-3c295-predictor-6f998d6587-44pj7.18d3c0c564f12816 namespace: kserve-ci-e2e-test resourceVersion: "22872" uid: 3991e893-ba95-47e6-baf8-8be9af06152a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c295-predictor-6f998d6587-44pj7 namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: 2f8c5e33-99fa-4e19-9286-eb3f5ec93e20 kind: Event lastTimestamp: "2026-09-09T20:23:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:51Z" name: error-404-isvc-3c295-predictor-6f998d6587-44pj7.18d3c0c5680a956e namespace: kserve-ci-e2e-test resourceVersion: "22874" uid: 2aaac007-df61-45a2-b1eb-f154b831315d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c295-predictor-6f998d6587-44pj7 namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: 2f8c5e33-99fa-4e19-9286-eb3f5ec93e20 kind: Event lastTimestamp: "2026-09-09T20:23:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:51Z" name: error-404-isvc-3c295-predictor-6f998d6587-44pj7.18d3c0c5687aa434 namespace: kserve-ci-e2e-test resourceVersion: "22875" uid: 7e705e3a-4d7c-4975-b3f9-1273ef1fc6f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c295-predictor-6f998d6587-44pj7 namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: 2f8c5e33-99fa-4e19-9286-eb3f5ec93e20 kind: Event lastTimestamp: "2026-09-09T20:23:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:51Z" name: error-404-isvc-3c295-predictor-6f998d6587-44pj7.18d3c0c56885d90d namespace: kserve-ci-e2e-test resourceVersion: "22876" uid: 4e57a68b-edf6-45eb-88f6-6540a0e2905a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c295-predictor-6f998d6587-44pj7 namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: 2f8c5e33-99fa-4e19-9286-eb3f5ec93e20 kind: Event lastTimestamp: "2026-09-09T20:23:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:51Z" name: error-404-isvc-3c295-predictor-6f998d6587-44pj7.18d3c0c56b3e47c8 namespace: kserve-ci-e2e-test resourceVersion: "22877" uid: 58c520fe-4df3-4e70-b7d2-4608f124eca4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c295-predictor-6f998d6587-44pj7 namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: 2f8c5e33-99fa-4e19-9286-eb3f5ec93e20 kind: Event lastTimestamp: "2026-09-09T20:23:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:51Z" name: error-404-isvc-3c295-predictor-6f998d6587-44pj7.18d3c0c56bbb7d1b namespace: kserve-ci-e2e-test resourceVersion: "22878" uid: e8baf28c-3e14-4827-9564-d207f19d1b8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:23:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c295-predictor-6f998d6587-44pj7 namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: 2f8c5e33-99fa-4e19-9286-eb3f5ec93e20 kind: Event lastTimestamp: "2026-09-09T20:24:28Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:28Z" name: error-404-isvc-3c295-predictor-6f998d6587-44pj7.18d3c0c5ab83c32a namespace: kserve-ci-e2e-test resourceVersion: "23219" uid: 595bd1a5-d001-4e86-ac54-72fb68af6f97 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3c295-predictor-6f998d6587-44pj7 namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: 2f8c5e33-99fa-4e19-9286-eb3f5ec93e20 kind: Event lastTimestamp: "2026-09-09T20:33:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:04Z" name: error-404-isvc-3c295-predictor-6f998d6587-44pj7.18d3c1464bb03594 namespace: kserve-ci-e2e-test resourceVersion: "27491" uid: e9b92cc1-f732-40d6-9181-9a6ae6045229 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3c295-predictor-6f998d6587-44pj7 namespace: kserve-ci-e2e-test resourceVersion: "22834" uid: 2f8c5e33-99fa-4e19-9286-eb3f5ec93e20 kind: Event lastTimestamp: "2026-09-09T20:33:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:04Z" name: error-404-isvc-3c295-predictor-6f998d6587-44pj7.18d3c1464bb17417 namespace: kserve-ci-e2e-test resourceVersion: "27493" uid: eca0f142-780d-48e9-b303-eff3885d210d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3c295-predictor-6f998d6587 namespace: kserve-ci-e2e-test resourceVersion: "22826" uid: a77480a8-330c-4c06-8175-94b044f5f5f9 kind: Event lastTimestamp: "2026-09-09T20:23:49Z" message: 'Created pod: error-404-isvc-3c295-predictor-6f998d6587-44pj7' metadata: creationTimestamp: "2026-09-09T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:49Z" name: error-404-isvc-3c295-predictor-6f998d6587.18d3c0c51d66bdd8 namespace: kserve-ci-e2e-test resourceVersion: "22833" uid: 1d437304-4fa1-4671-bf92-39cc50907581 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3c295-predictor namespace: kserve-ci-e2e-test resourceVersion: "22824" uid: e8e4d23d-6a23-4a58-b002-676df8554982 kind: Event lastTimestamp: "2026-09-09T20:23:49Z" message: Scaled up replica set error-404-isvc-3c295-predictor-6f998d6587 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:49Z" name: error-404-isvc-3c295-predictor.18d3c0c51bbe5986 namespace: kserve-ci-e2e-test resourceVersion: "22827" uid: 283d0b1f-73e9-471b-9ae7-3fe4f8106dcd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:24:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c295-predictor namespace: kserve-ci-e2e-test resourceVersion: "22825" uid: c6c9bbb0-ee3f-4f7d-8fe3-c51aaaffb763 kind: Event lastTimestamp: "2026-09-09T20:24:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:19Z" name: error-404-isvc-3c295-predictor.18d3c0c89a3483e4 namespace: kserve-ci-e2e-test resourceVersion: "23148" uid: 00336dcd-8e36-4b80-8295-c64ab7f0a443 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:24:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c295-predictor namespace: kserve-ci-e2e-test resourceVersion: "22825" uid: c6c9bbb0-ee3f-4f7d-8fe3-c51aaaffb763 kind: Event lastTimestamp: "2026-09-09T20:24:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:19Z" name: error-404-isvc-3c295-predictor.18d3c0c89a98c635 namespace: kserve-ci-e2e-test resourceVersion: "23149" uid: 8355a433-40f4-460f-850c-2ed9e85325d1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:24:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c295-predictor namespace: kserve-ci-e2e-test resourceVersion: "23014" uid: c6c9bbb0-ee3f-4f7d-8fe3-c51aaaffb763 kind: Event lastTimestamp: "2026-09-09T20:25:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:25:04Z" name: error-404-isvc-3c295-predictor.18d3c0cf97e044e6 namespace: kserve-ci-e2e-test resourceVersion: "23749" uid: 6f1bb529-f07e-4bd8-ab3c-86d8c8f32822 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:24:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3c295-predictor namespace: kserve-ci-e2e-test resourceVersion: "23014" uid: c6c9bbb0-ee3f-4f7d-8fe3-c51aaaffb763 kind: Event lastTimestamp: "2026-09-09T20:25:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:25:04Z" name: error-404-isvc-3c295-predictor.18d3c0cf98420de1 namespace: kserve-ci-e2e-test resourceVersion: "23750" uid: b9095f8f-bb89-4292-8c4e-8fd79b9f2e2a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:23:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3c295 namespace: kserve-ci-e2e-test resourceVersion: "22813" uid: f576fdb5-2804-42f9-84fd-e0b8e90fbf15 kind: Event lastTimestamp: "2026-09-09T20:24:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:24:38Z" name: error-404-isvc-3c295.18d3c0c51bc7ea59 namespace: kserve-ci-e2e-test resourceVersion: "23349" uid: fabda17e-2c3e-4b9f-943c-97427a4241ae reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3c295 namespace: kserve-ci-e2e-test resourceVersion: "23346" uid: f576fdb5-2804-42f9-84fd-e0b8e90fbf15 kind: Event lastTimestamp: "2026-09-09T20:24:38Z" message: InferenceService [error-404-isvc-3c295] is Ready metadata: creationTimestamp: "2026-09-09T20:24:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:24:38Z" name: error-404-isvc-3c295.18d3c0d0695112f6 namespace: kserve-ci-e2e-test resourceVersion: "23347" uid: 301cae80-a35b-4902-8144-fdd45cd10f0a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:34:00.618033Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-54686-predictor-5478745645-rdwpn namespace: kserve-ci-e2e-test resourceVersion: "28071" uid: 7150978e-3939-47c3-bc39-626cd5d3fb92 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-54686-predictor-5478745645-rdwpn to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:34:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:34:00Z" name: error-404-isvc-54686-predictor-5478745645-rdwpn.18d3c153532bc226 namespace: kserve-ci-e2e-test resourceVersion: "28075" uid: 1025fdae-9ac2-4637-9a13-b45fbb1a37f1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-54686-predictor-5478745645-rdwpn namespace: kserve-ci-e2e-test resourceVersion: "28081" uid: 7150978e-3939-47c3-bc39-626cd5d3fb92 kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:34:01Z" name: error-404-isvc-54686-predictor-5478745645-rdwpn.18d3c15375248aa6 namespace: kserve-ci-e2e-test resourceVersion: "28100" uid: 57edd286-805f-42f7-b61b-ca9d5932fea8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54686-predictor-5478745645-rdwpn namespace: kserve-ci-e2e-test resourceVersion: "28073" uid: 7150978e-3939-47c3-bc39-626cd5d3fb92 kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:01Z" name: error-404-isvc-54686-predictor-5478745645-rdwpn.18d3c15377109d80 namespace: kserve-ci-e2e-test resourceVersion: "28102" uid: d4d72cf8-ed99-4d00-b478-685d07777865 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54686-predictor-5478745645-rdwpn namespace: kserve-ci-e2e-test resourceVersion: "28073" uid: 7150978e-3939-47c3-bc39-626cd5d3fb92 kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:01Z" name: error-404-isvc-54686-predictor-5478745645-rdwpn.18d3c15379b43e4d namespace: kserve-ci-e2e-test resourceVersion: "28105" uid: 2ec426f6-4d00-440c-b475-9802d8b7ba96 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54686-predictor-5478745645-rdwpn namespace: kserve-ci-e2e-test resourceVersion: "28073" uid: 7150978e-3939-47c3-bc39-626cd5d3fb92 kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:01Z" name: error-404-isvc-54686-predictor-5478745645-rdwpn.18d3c1537a261dc6 namespace: kserve-ci-e2e-test resourceVersion: "28106" uid: 4636098b-8e95-4b80-a832-bc5881b7a060 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-54686-predictor-5478745645-rdwpn namespace: kserve-ci-e2e-test resourceVersion: "28073" uid: 7150978e-3939-47c3-bc39-626cd5d3fb92 kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:01Z" name: error-404-isvc-54686-predictor-5478745645-rdwpn.18d3c1537a301672 namespace: kserve-ci-e2e-test resourceVersion: "28107" uid: 6a163f38-c8de-4f81-83e9-30e1ff652bdc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-54686-predictor-5478745645-rdwpn namespace: kserve-ci-e2e-test resourceVersion: "28073" uid: 7150978e-3939-47c3-bc39-626cd5d3fb92 kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:01Z" name: error-404-isvc-54686-predictor-5478745645-rdwpn.18d3c1537d1b5b17 namespace: kserve-ci-e2e-test resourceVersion: "28108" uid: 77cb77b6-acdc-4023-8820-91ccb7565837 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-54686-predictor-5478745645-rdwpn namespace: kserve-ci-e2e-test resourceVersion: "28073" uid: 7150978e-3939-47c3-bc39-626cd5d3fb92 kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:01Z" name: error-404-isvc-54686-predictor-5478745645-rdwpn.18d3c1537d867071 namespace: kserve-ci-e2e-test resourceVersion: "28109" uid: ee665c0c-7b85-4f5f-b3e3-b3f157fd2788 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54686-predictor-5478745645-rdwpn namespace: kserve-ci-e2e-test resourceVersion: "28073" uid: 7150978e-3939-47c3-bc39-626cd5d3fb92 kind: Event lastTimestamp: "2026-09-09T20:34:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:37Z" name: error-404-isvc-54686-predictor-5478745645-rdwpn.18d3c15392066b86 namespace: kserve-ci-e2e-test resourceVersion: "28610" uid: 329f67d2-914f-45f8-bb1c-f16d7a7604e4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:51:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-54686-predictor-5478745645-rdwpn namespace: kserve-ci-e2e-test resourceVersion: "28073" uid: 7150978e-3939-47c3-bc39-626cd5d3fb92 kind: Event lastTimestamp: "2026-09-09T20:51:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:51:29Z" name: error-404-isvc-54686-predictor-5478745645-rdwpn.18d3c2479bc51ac4 namespace: kserve-ci-e2e-test resourceVersion: "35838" uid: 3ec53874-cdae-4477-b0ea-4689edfbea97 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:51:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-54686-predictor-5478745645-rdwpn namespace: kserve-ci-e2e-test resourceVersion: "28073" uid: 7150978e-3939-47c3-bc39-626cd5d3fb92 kind: Event lastTimestamp: "2026-09-09T20:51:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:51:29Z" name: error-404-isvc-54686-predictor-5478745645-rdwpn.18d3c2479bc637ef namespace: kserve-ci-e2e-test resourceVersion: "35839" uid: 5aa85ab2-9b48-4e60-91dd-2db9d979fd90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-54686-predictor-5478745645 namespace: kserve-ci-e2e-test resourceVersion: "28064" uid: 91658c99-408d-4061-a950-5332d602e06c kind: Event lastTimestamp: "2026-09-09T20:34:00Z" message: 'Created pod: error-404-isvc-54686-predictor-5478745645-rdwpn' metadata: creationTimestamp: "2026-09-09T20:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:00Z" name: error-404-isvc-54686-predictor-5478745645.18d3c15352a2a24e namespace: kserve-ci-e2e-test resourceVersion: "28072" uid: 3b0b063c-f263-47b4-9406-680a7716a607 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-54686-predictor namespace: kserve-ci-e2e-test resourceVersion: "28062" uid: 799fb8ce-5242-4945-8b8d-147beb557a8f kind: Event lastTimestamp: "2026-09-09T20:34:00Z" message: Scaled up replica set error-404-isvc-54686-predictor-5478745645 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:00Z" name: error-404-isvc-54686-predictor.18d3c153511258b1 namespace: kserve-ci-e2e-test resourceVersion: "28065" uid: 53ffba4b-16b3-4c49-a632-1aae7d300992 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-54686-predictor namespace: kserve-ci-e2e-test resourceVersion: "28063" uid: 110b4996-9e4d-4e76-9923-7791c2761255 kind: Event lastTimestamp: "2026-09-09T20:34:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:30Z" name: error-404-isvc-54686-predictor.18d3c156cf9c744f namespace: kserve-ci-e2e-test resourceVersion: "28554" uid: e8305f92-18e7-42cb-afd7-f5e0cdf2c79f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-54686-predictor namespace: kserve-ci-e2e-test resourceVersion: "28063" uid: 110b4996-9e4d-4e76-9923-7791c2761255 kind: Event lastTimestamp: "2026-09-09T20:34:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:30Z" name: error-404-isvc-54686-predictor.18d3c156cfffb3ca namespace: kserve-ci-e2e-test resourceVersion: "28555" uid: 309d5bdd-69a7-4432-9049-67d8e1ce25d7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:34:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-54686-predictor namespace: kserve-ci-e2e-test resourceVersion: "28397" uid: 110b4996-9e4d-4e76-9923-7791c2761255 kind: Event lastTimestamp: "2026-09-09T20:35:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:15Z" name: error-404-isvc-54686-predictor.18d3c15dcd6332fd namespace: kserve-ci-e2e-test resourceVersion: "28978" uid: fa00def9-9794-4147-ae24-4e0dfab5b1b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:34:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-54686-predictor namespace: kserve-ci-e2e-test resourceVersion: "28397" uid: 110b4996-9e4d-4e76-9923-7791c2761255 kind: Event lastTimestamp: "2026-09-09T20:35:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:15Z" name: error-404-isvc-54686-predictor.18d3c15dcdc3ebee namespace: kserve-ci-e2e-test resourceVersion: "28979" uid: 08d0007a-bcfb-4153-8af5-391d57f53947 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:34:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-54686 namespace: kserve-ci-e2e-test resourceVersion: "28046" uid: cda399bf-6374-4ead-bdf9-6d5a75b06473 kind: Event lastTimestamp: "2026-09-09T20:34:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:34:47Z" name: error-404-isvc-54686.18d3c153511a270f namespace: kserve-ci-e2e-test resourceVersion: "28708" uid: 8f34d476-be83-4588-8fc7-e0b8582181bd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-54686 namespace: kserve-ci-e2e-test resourceVersion: "28706" uid: cda399bf-6374-4ead-bdf9-6d5a75b06473 kind: Event lastTimestamp: "2026-09-09T20:34:47Z" message: InferenceService [error-404-isvc-54686] is Ready metadata: creationTimestamp: "2026-09-09T20:34:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:34:47Z" name: error-404-isvc-54686.18d3c15e4d588e0f namespace: kserve-ci-e2e-test resourceVersion: "28707" uid: 65d2a9af-611d-4cfd-b5d9-3c1d180332d1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:33:04.711396Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn namespace: kserve-ci-e2e-test resourceVersion: "27502" uid: f0a2f6b7-48cb-4a1e-a91c-3fc43aa6aa16 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-97f7b-predictor-5c667894df-2bfzn to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:33:04Z" name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn.18d3c1464ee02e5d namespace: kserve-ci-e2e-test resourceVersion: "27506" uid: 29e976f4-86e1-4d51-a58f-20795f216884 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn namespace: kserve-ci-e2e-test resourceVersion: "27512" uid: f0a2f6b7-48cb-4a1e-a91c-3fc43aa6aa16 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:33:05Z" name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn.18d3c14670d4b480 namespace: kserve-ci-e2e-test resourceVersion: "27527" uid: 3358f05f-8ce9-4f0e-965f-53cc5a9fb1e9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: f0a2f6b7-48cb-4a1e-a91c-3fc43aa6aa16 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:05Z" name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn.18d3c146724cb23d namespace: kserve-ci-e2e-test resourceVersion: "27529" uid: ebac5fe5-7917-43c8-820c-2d5f50ae8630 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: f0a2f6b7-48cb-4a1e-a91c-3fc43aa6aa16 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:05Z" name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn.18d3c14674c24543 namespace: kserve-ci-e2e-test resourceVersion: "27532" uid: 9f18d6d7-99f3-48ea-bade-5da35939ef6e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: f0a2f6b7-48cb-4a1e-a91c-3fc43aa6aa16 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:05Z" name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn.18d3c1467538c9d3 namespace: kserve-ci-e2e-test resourceVersion: "27533" uid: 891104e6-11f3-4a2e-9e04-f310d5f922c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: f0a2f6b7-48cb-4a1e-a91c-3fc43aa6aa16 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:05Z" name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn.18d3c146754361e2 namespace: kserve-ci-e2e-test resourceVersion: "27534" uid: 7302c1e0-eca8-4a45-bde9-365740ba4ef3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: f0a2f6b7-48cb-4a1e-a91c-3fc43aa6aa16 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:05Z" name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn.18d3c1467839fcd4 namespace: kserve-ci-e2e-test resourceVersion: "27535" uid: c827dfc4-fd54-47c2-8dab-3f7a396299d0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: f0a2f6b7-48cb-4a1e-a91c-3fc43aa6aa16 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:05Z" name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn.18d3c14678aec640 namespace: kserve-ci-e2e-test resourceVersion: "27536" uid: 93c3ac1f-a597-403f-a8b9-a046c96b5000 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: f0a2f6b7-48cb-4a1e-a91c-3fc43aa6aa16 kind: Event lastTimestamp: "2026-09-09T20:33:41Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:41Z" name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn.18d3c146799cf2fc namespace: kserve-ci-e2e-test resourceVersion: "27827" uid: df1170ef-a07a-4ae3-89dd-598ca99a5bbf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: f0a2f6b7-48cb-4a1e-a91c-3fc43aa6aa16 kind: Event lastTimestamp: "2026-09-09T20:34:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:14Z" name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn.18d3c156a3524b7a namespace: kserve-ci-e2e-test resourceVersion: "28324" uid: feb7436d-bc6c-41f0-adcd-3337825b5aab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: f0a2f6b7-48cb-4a1e-a91c-3fc43aa6aa16 kind: Event lastTimestamp: "2026-09-09T20:34:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:14Z" name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn.18d3c156a3566f58 namespace: kserve-ci-e2e-test resourceVersion: "28326" uid: 6229bb83-c70e-4675-b054-3c98a3675725 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: f0a2f6b7-48cb-4a1e-a91c-3fc43aa6aa16 kind: Event lastTimestamp: "2026-09-09T20:34:16Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": dial tcp 10.132.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:16Z" name: error-404-isvc-97f7b-predictor-5c667894df-2bfzn.18d3c15701ca78ff namespace: kserve-ci-e2e-test resourceVersion: "28425" uid: 1ba81f19-558b-4079-b4cf-e0e0f420843e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-97f7b-predictor-5c667894df namespace: kserve-ci-e2e-test resourceVersion: "27494" uid: 7ac69e01-0785-4a68-952d-9660ac4787df kind: Event lastTimestamp: "2026-09-09T20:33:04Z" message: 'Created pod: error-404-isvc-97f7b-predictor-5c667894df-2bfzn' metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:33:04Z" name: error-404-isvc-97f7b-predictor-5c667894df.18d3c1464e6e25fb namespace: kserve-ci-e2e-test resourceVersion: "27503" uid: c35dbc75-0a28-4953-892a-bc21c19c3464 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-97f7b-predictor namespace: kserve-ci-e2e-test resourceVersion: "27489" uid: b8c3bf30-024e-4115-a897-9cb3ed775a2d kind: Event lastTimestamp: "2026-09-09T20:33:04Z" message: Scaled up replica set error-404-isvc-97f7b-predictor-5c667894df from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:33:04Z" name: error-404-isvc-97f7b-predictor.18d3c1464c990d8e namespace: kserve-ci-e2e-test resourceVersion: "27497" uid: 4eb465e1-1b7f-4725-970d-67714b155d0f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-97f7b-predictor namespace: kserve-ci-e2e-test resourceVersion: "27492" uid: 8e03b8ea-2edb-42ce-8ea2-3f2555cfc48b kind: Event lastTimestamp: "2026-09-09T20:33:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:33:19Z" name: error-404-isvc-97f7b-predictor.18d3c149cb0782fa namespace: kserve-ci-e2e-test resourceVersion: "27662" uid: 8b3a38ba-b022-4070-884a-f35f7c093790 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-97f7b-predictor namespace: kserve-ci-e2e-test resourceVersion: "27492" uid: 8e03b8ea-2edb-42ce-8ea2-3f2555cfc48b kind: Event lastTimestamp: "2026-09-09T20:33:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:33:19Z" name: error-404-isvc-97f7b-predictor.18d3c149cb4f4ddb namespace: kserve-ci-e2e-test resourceVersion: "27664" uid: 0f8d3561-ac10-4297-8880-9e33fff3e35b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:33:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-97f7b-predictor namespace: kserve-ci-e2e-test resourceVersion: "27663" uid: 8e03b8ea-2edb-42ce-8ea2-3f2555cfc48b kind: Event lastTimestamp: "2026-09-09T20:34:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:33:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:04Z" name: error-404-isvc-97f7b-predictor.18d3c14d49f82234 namespace: kserve-ci-e2e-test resourceVersion: "28180" uid: 5acafcd0-92a6-432d-8d67-bc6630c347c3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:33:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-97f7b-predictor namespace: kserve-ci-e2e-test resourceVersion: "27663" uid: 8e03b8ea-2edb-42ce-8ea2-3f2555cfc48b kind: Event lastTimestamp: "2026-09-09T20:34:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:33:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:04Z" name: error-404-isvc-97f7b-predictor.18d3c14d4a595830 namespace: kserve-ci-e2e-test resourceVersion: "28182" uid: abecc584-bfda-46a9-a50d-6ef0ee5b17d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-97f7b namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: 8f1f9d6c-6af5-4762-b91a-f8d9c50bcacb kind: Event lastTimestamp: "2026-09-09T20:33:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:33:51Z" name: error-404-isvc-97f7b.18d3c1464c9bacc3 namespace: kserve-ci-e2e-test resourceVersion: "27912" uid: 6cebdb84-5f7d-491f-878a-88168ff031f1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-97f7b namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: 8f1f9d6c-6af5-4762-b91a-f8d9c50bcacb kind: Event lastTimestamp: "2026-09-09T20:33:04Z" message: 'Failed to update status for InferenceService "error-404-isvc-97f7b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-97f7b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:33:04Z" name: error-404-isvc-97f7b.18d3c1464d5bfd14 namespace: kserve-ci-e2e-test resourceVersion: "27499" uid: d79f9029-0c97-4ef4-8537-896719c59186 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-97f7b namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: 8f1f9d6c-6af5-4762-b91a-f8d9c50bcacb kind: Event lastTimestamp: "2026-09-09T20:33:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-97f7b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:33:04Z" name: error-404-isvc-97f7b.18d3c1464d5c4a43 namespace: kserve-ci-e2e-test resourceVersion: "27500" uid: 142516c9-3804-4ff3-8825-fd3724095dd5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-97f7b namespace: kserve-ci-e2e-test resourceVersion: "27910" uid: 8f1f9d6c-6af5-4762-b91a-f8d9c50bcacb kind: Event lastTimestamp: "2026-09-09T20:33:51Z" message: InferenceService [error-404-isvc-97f7b] is Ready metadata: creationTimestamp: "2026-09-09T20:33:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:33:51Z" name: error-404-isvc-97f7b.18d3c151351d0267 namespace: kserve-ci-e2e-test resourceVersion: "27911" uid: 6a5820c7-ae44-4f80-a1fc-b36c25bab717 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:34:15.043414Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh namespace: kserve-ci-e2e-test resourceVersion: "28364" uid: 7ae33901-ff0d-4f8a-b058-c56f41c2f828 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:34:15Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh.18d3c156aefd9d75 namespace: kserve-ci-e2e-test resourceVersion: "28369" uid: 7b615e54-5980-42a2-91af-98df39e61bc8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh namespace: kserve-ci-e2e-test resourceVersion: "28367" uid: 7ae33901-ff0d-4f8a-b058-c56f41c2f828 kind: Event lastTimestamp: "2026-09-09T20:34:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-b9ccb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:15Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh.18d3c156bdf0709f namespace: kserve-ci-e2e-test resourceVersion: "28383" uid: 814d9d9a-95d6-441f-9902-af96cdb4c638 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh namespace: kserve-ci-e2e-test resourceVersion: "28374" uid: 7ae33901-ff0d-4f8a-b058-c56f41c2f828 kind: Event lastTimestamp: "2026-09-09T20:34:16Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:34:16Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh.18d3c156f499a5f6 namespace: kserve-ci-e2e-test resourceVersion: "28414" uid: 08d671f5-be03-48e6-9f65-22215a022216 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh namespace: kserve-ci-e2e-test resourceVersion: "28367" uid: 7ae33901-ff0d-4f8a-b058-c56f41c2f828 kind: Event lastTimestamp: "2026-09-09T20:34:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:16Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh.18d3c156f5d97e06 namespace: kserve-ci-e2e-test resourceVersion: "28416" uid: fb22bfd1-a859-4483-b89f-59b3ff0cbd74 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh namespace: kserve-ci-e2e-test resourceVersion: "28367" uid: 7ae33901-ff0d-4f8a-b058-c56f41c2f828 kind: Event lastTimestamp: "2026-09-09T20:34:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:16Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh.18d3c156f8dd1b52 namespace: kserve-ci-e2e-test resourceVersion: "28418" uid: d7bafb52-f149-4de4-a8f7-532f244c962d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh namespace: kserve-ci-e2e-test resourceVersion: "28367" uid: 7ae33901-ff0d-4f8a-b058-c56f41c2f828 kind: Event lastTimestamp: "2026-09-09T20:34:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:16Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh.18d3c156f94b5c04 namespace: kserve-ci-e2e-test resourceVersion: "28419" uid: c510ba99-5bba-4ba3-89f6-666c2846f6e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh namespace: kserve-ci-e2e-test resourceVersion: "28367" uid: 7ae33901-ff0d-4f8a-b058-c56f41c2f828 kind: Event lastTimestamp: "2026-09-09T20:34:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:16Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh.18d3c156f9554d6f namespace: kserve-ci-e2e-test resourceVersion: "28420" uid: 8e360bc8-09f2-4382-a542-5941ada04089 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh namespace: kserve-ci-e2e-test resourceVersion: "28367" uid: 7ae33901-ff0d-4f8a-b058-c56f41c2f828 kind: Event lastTimestamp: "2026-09-09T20:34:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:16Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh.18d3c156fc5e1d03 namespace: kserve-ci-e2e-test resourceVersion: "28421" uid: aa04c83f-3962-4a7d-8bbe-6109de457b07 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh namespace: kserve-ci-e2e-test resourceVersion: "28367" uid: 7ae33901-ff0d-4f8a-b058-c56f41c2f828 kind: Event lastTimestamp: "2026-09-09T20:34:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:16Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh.18d3c156fd29522b namespace: kserve-ci-e2e-test resourceVersion: "28422" uid: 4286f0ad-e2ff-4a00-af19-ea9e7daab7db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-09T20:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh namespace: kserve-ci-e2e-test resourceVersion: "28367" uid: 7ae33901-ff0d-4f8a-b058-c56f41c2f828 kind: Event lastTimestamp: "2026-09-09T20:43:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:43:42Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh.18d3c1571555aafe namespace: kserve-ci-e2e-test resourceVersion: "32595" uid: d5d68008-8cba-4649-8002-c863c9eb60eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh namespace: kserve-ci-e2e-test resourceVersion: "28367" uid: 7ae33901-ff0d-4f8a-b058-c56f41c2f828 kind: Event lastTimestamp: "2026-09-09T20:43:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:43:39Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh.18d3c1da27c07e77 namespace: kserve-ci-e2e-test resourceVersion: "32566" uid: fe1bc3dd-4258-4234-a489-63418f43ee10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh namespace: kserve-ci-e2e-test resourceVersion: "28367" uid: 7ae33901-ff0d-4f8a-b058-c56f41c2f828 kind: Event lastTimestamp: "2026-09-09T20:43:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:43:39Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh.18d3c1da27c0f3cd namespace: kserve-ci-e2e-test resourceVersion: "32567" uid: 1b2f1845-8d28-46c0-b38e-5f7b590fb570 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:43:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh namespace: kserve-ci-e2e-test resourceVersion: "28367" uid: 7ae33901-ff0d-4f8a-b058-c56f41c2f828 kind: Event lastTimestamp: "2026-09-09T20:43:42Z" message: 'Readiness probe failed: Get "https://10.132.0.36:8643/healthz": dial tcp 10.132.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:43:42Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh.18d3c1dadddf18ea namespace: kserve-ci-e2e-test resourceVersion: "32594" uid: 6889e8be-2a5f-446a-8ac6-c04b4f245cf0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b9ccb-predictor-d5c8df47c namespace: kserve-ci-e2e-test resourceVersion: "28358" uid: 7c71a57b-c124-4e97-981f-ff904a55dc80 kind: Event lastTimestamp: "2026-09-09T20:34:15Z" message: 'Created pod: error-404-isvc-b9ccb-predictor-d5c8df47c-x72vh' metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:15Z" name: error-404-isvc-b9ccb-predictor-d5c8df47c.18d3c156ae9b65da namespace: kserve-ci-e2e-test resourceVersion: "28366" uid: 8ca4ac00-130f-4796-9ad6-f75bb27c83c8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b9ccb-predictor namespace: kserve-ci-e2e-test resourceVersion: "28356" uid: 4d847fc2-c473-4d0c-af18-6816a1af7a54 kind: Event lastTimestamp: "2026-09-09T20:34:15Z" message: Scaled up replica set error-404-isvc-b9ccb-predictor-d5c8df47c from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:15Z" name: error-404-isvc-b9ccb-predictor.18d3c156acd05d5d namespace: kserve-ci-e2e-test resourceVersion: "28359" uid: 5a792d02-a8a2-4944-bace-08ff1ae02c8b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:34:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9ccb-predictor namespace: kserve-ci-e2e-test resourceVersion: "28357" uid: eaf66c95-e383-4c45-a03b-1c86e0d66ed7 kind: Event lastTimestamp: "2026-09-09T20:35:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:34:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:45Z" name: error-404-isvc-b9ccb-predictor.18d3c15a2b3e470a namespace: kserve-ci-e2e-test resourceVersion: "29249" uid: 46e2fdf6-0242-494f-bd85-1bcb66d08454 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:34:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9ccb-predictor namespace: kserve-ci-e2e-test resourceVersion: "28357" uid: eaf66c95-e383-4c45-a03b-1c86e0d66ed7 kind: Event lastTimestamp: "2026-09-09T20:35:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:34:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:45Z" name: error-404-isvc-b9ccb-predictor.18d3c15a2bc0cab1 namespace: kserve-ci-e2e-test resourceVersion: "29250" uid: 313a9ea1-ef82-49b8-8dca-01001bf1b759 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9ccb-predictor namespace: kserve-ci-e2e-test resourceVersion: "28547" uid: eaf66c95-e383-4c45-a03b-1c86e0d66ed7 kind: Event lastTimestamp: "2026-09-09T20:34:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:45Z" name: error-404-isvc-b9ccb-predictor.18d3c15daa6816ad namespace: kserve-ci-e2e-test resourceVersion: "28674" uid: 214285ba-5470-4692-9e3e-df6487a82029 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b9ccb-predictor namespace: kserve-ci-e2e-test resourceVersion: "28547" uid: eaf66c95-e383-4c45-a03b-1c86e0d66ed7 kind: Event lastTimestamp: "2026-09-09T20:34:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:45Z" name: error-404-isvc-b9ccb-predictor.18d3c15daaebab46 namespace: kserve-ci-e2e-test resourceVersion: "28677" uid: a2ad476d-436c-4935-b640-9fcf92633835 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9ccb namespace: kserve-ci-e2e-test resourceVersion: "28346" uid: b9c8ac09-151d-44b0-9232-b7f8131aa841 kind: Event lastTimestamp: "2026-09-09T20:35:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:35:02Z" name: error-404-isvc-b9ccb.18d3c156acd74169 namespace: kserve-ci-e2e-test resourceVersion: "28840" uid: 262e43ec-b1f7-46d9-834a-23c1d23d7513 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b9ccb namespace: kserve-ci-e2e-test resourceVersion: "28838" uid: b9c8ac09-151d-44b0-9232-b7f8131aa841 kind: Event lastTimestamp: "2026-09-09T20:35:02Z" message: InferenceService [error-404-isvc-b9ccb] is Ready metadata: creationTimestamp: "2026-09-09T20:35:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:35:02Z" name: error-404-isvc-b9ccb.18d3c161d0e7088c namespace: kserve-ci-e2e-test resourceVersion: "28839" uid: fde9e841-7778-418a-9def-b9a23204acf4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:23:25.461401Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5 namespace: kserve-ci-e2e-test resourceVersion: "22442" uid: 77ceac87-95c5-40e6-a9b1-d13275442279 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5 to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:23:25Z" name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5.18d3c0bf70e2bebe namespace: kserve-ci-e2e-test resourceVersion: "22446" uid: 5d0c728b-c579-4f84-900b-8c69871edcd7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5 namespace: kserve-ci-e2e-test resourceVersion: "22450" uid: 77ceac87-95c5-40e6-a9b1-d13275442279 kind: Event lastTimestamp: "2026-09-09T20:23:26Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:23:26Z" name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5.18d3c0bf92d02929 namespace: kserve-ci-e2e-test resourceVersion: "22470" uid: bc590f49-69e0-4530-8784-73f8f1b366cf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: 77ceac87-95c5-40e6-a9b1-d13275442279 kind: Event lastTimestamp: "2026-09-09T20:23:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:26Z" name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5.18d3c0bf9435db51 namespace: kserve-ci-e2e-test resourceVersion: "22473" uid: b4d1ab32-f651-42c7-8c39-a2adec7b7ce2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: 77ceac87-95c5-40e6-a9b1-d13275442279 kind: Event lastTimestamp: "2026-09-09T20:23:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:26Z" name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5.18d3c0bf9701b078 namespace: kserve-ci-e2e-test resourceVersion: "22477" uid: 9ba1a303-bc1d-4d8b-8a29-277ebc3f1b71 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: 77ceac87-95c5-40e6-a9b1-d13275442279 kind: Event lastTimestamp: "2026-09-09T20:23:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:26Z" name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5.18d3c0bf97b3768d namespace: kserve-ci-e2e-test resourceVersion: "22480" uid: cebad47f-8a96-49af-ba5d-1062b8b177e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: 77ceac87-95c5-40e6-a9b1-d13275442279 kind: Event lastTimestamp: "2026-09-09T20:23:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:26Z" name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5.18d3c0bf97bbe7f5 namespace: kserve-ci-e2e-test resourceVersion: "22481" uid: e926208c-e347-4fa3-a841-872753281e84 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: 77ceac87-95c5-40e6-a9b1-d13275442279 kind: Event lastTimestamp: "2026-09-09T20:23:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:26Z" name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5.18d3c0bf9b632bf7 namespace: kserve-ci-e2e-test resourceVersion: "22483" uid: afc1a669-e2c7-4a9b-a1d2-7bb63e24cc02 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: 77ceac87-95c5-40e6-a9b1-d13275442279 kind: Event lastTimestamp: "2026-09-09T20:23:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:26Z" name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5.18d3c0bf9bd7546e namespace: kserve-ci-e2e-test resourceVersion: "22484" uid: 14abf3fb-10a0-4533-9ba2-eb258d6dc0e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:23:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: 77ceac87-95c5-40e6-a9b1-d13275442279 kind: Event lastTimestamp: "2026-09-09T20:24:04Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:04Z" name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5.18d3c0c00e345437 namespace: kserve-ci-e2e-test resourceVersion: "23001" uid: 9a86241f-561c-47e6-949f-b7e1149ee7e7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: 77ceac87-95c5-40e6-a9b1-d13275442279 kind: Event lastTimestamp: "2026-09-09T20:24:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:24:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:45Z" name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5.18d3c0d2178c278e namespace: kserve-ci-e2e-test resourceVersion: "23464" uid: e86626b8-bbeb-49f2-bb76-24961af50fd4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: 77ceac87-95c5-40e6-a9b1-d13275442279 kind: Event lastTimestamp: "2026-09-09T20:24:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:24:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:45Z" name: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5.18d3c0d2178cb1c0 namespace: kserve-ci-e2e-test resourceVersion: "23465" uid: f4592c77-969c-4239-af7e-da3e6fa8cc72 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bf2e8-predictor-68cb795cb9 namespace: kserve-ci-e2e-test resourceVersion: "22435" uid: 803b6654-03fa-423f-9910-1900de0a1bc0 kind: Event lastTimestamp: "2026-09-09T20:23:25Z" message: 'Created pod: error-404-isvc-bf2e8-predictor-68cb795cb9-4s2h5' metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:25Z" name: error-404-isvc-bf2e8-predictor-68cb795cb9.18d3c0bf70728102 namespace: kserve-ci-e2e-test resourceVersion: "22444" uid: a421639a-9a93-4e13-8add-f9e4bc657f6c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bf2e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22433" uid: 7d4280eb-4f88-4270-a484-483a5323dc78 kind: Event lastTimestamp: "2026-09-09T20:23:25Z" message: Scaled up replica set error-404-isvc-bf2e8-predictor-68cb795cb9 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:25Z" name: error-404-isvc-bf2e8-predictor.18d3c0bf6e68d975 namespace: kserve-ci-e2e-test resourceVersion: "22436" uid: 06c91d2b-dfc7-4b48-85e3-e6a86b983943 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:23:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bf2e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22434" uid: 88c5fd5b-b61f-41a5-91ee-30d8bfe1d59b kind: Event lastTimestamp: "2026-09-09T20:24:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:10Z" name: error-404-isvc-bf2e8-predictor.18d3c0c2ece82bfe namespace: kserve-ci-e2e-test resourceVersion: "23060" uid: 06dba665-7eeb-4acd-944c-b285103793e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:23:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bf2e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22434" uid: 88c5fd5b-b61f-41a5-91ee-30d8bfe1d59b kind: Event lastTimestamp: "2026-09-09T20:24:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:10Z" name: error-404-isvc-bf2e8-predictor.18d3c0c2ed4f9032 namespace: kserve-ci-e2e-test resourceVersion: "23061" uid: 6ac939b5-88d2-43b2-86ac-b8b6a4d0c754 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:23:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bf2e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22657" uid: 88c5fd5b-b61f-41a5-91ee-30d8bfe1d59b kind: Event lastTimestamp: "2026-09-09T20:24:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:40Z" name: error-404-isvc-bf2e8-predictor.18d3c0c66bf0b1b6 namespace: kserve-ci-e2e-test resourceVersion: "23365" uid: 707133f2-2e5a-44b1-95ae-6e744ad6bcdc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:23:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bf2e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22657" uid: 88c5fd5b-b61f-41a5-91ee-30d8bfe1d59b kind: Event lastTimestamp: "2026-09-09T20:24:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:40Z" name: error-404-isvc-bf2e8-predictor.18d3c0c66c52a74e namespace: kserve-ci-e2e-test resourceVersion: "23366" uid: cd11f5a6-22c7-4289-8353-31c58c5925e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bf2e8 namespace: kserve-ci-e2e-test resourceVersion: "22419" uid: b5bf20e2-5d56-48f8-91fd-b5186c73ff3c kind: Event lastTimestamp: "2026-09-09T20:24:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:24:14Z" name: error-404-isvc-bf2e8.18d3c0bf6e6f375d namespace: kserve-ci-e2e-test resourceVersion: "23098" uid: 7ffd01a4-827f-4c36-af7e-8d607133372b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bf2e8 namespace: kserve-ci-e2e-test resourceVersion: "23095" uid: b5bf20e2-5d56-48f8-91fd-b5186c73ff3c kind: Event lastTimestamp: "2026-09-09T20:24:14Z" message: InferenceService [error-404-isvc-bf2e8] is Ready metadata: creationTimestamp: "2026-09-09T20:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:24:14Z" name: error-404-isvc-bf2e8.18d3c0caca1d3567 namespace: kserve-ci-e2e-test resourceVersion: "23096" uid: 307b672a-e7a2-4d0c-9594-20083d488d02 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:13:24.765375Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d2727-predictor-6487c46c67-cphk6 namespace: kserve-ci-e2e-test resourceVersion: "17553" uid: 71cbc44b-aea9-4c53-85c7-7d03e61d0e42 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d2727-predictor-6487c46c67-cphk6 to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:13:24Z" name: error-404-isvc-d2727-predictor-6487c46c67-cphk6.18d3c033949cb63d namespace: kserve-ci-e2e-test resourceVersion: "17558" uid: 55060606-a809-495a-afcb-3a8caa13abb0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d2727-predictor-6487c46c67-cphk6 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: 71cbc44b-aea9-4c53-85c7-7d03e61d0e42 kind: Event lastTimestamp: "2026-09-09T20:13:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d2727-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:24Z" name: error-404-isvc-d2727-predictor-6487c46c67-cphk6.18d3c033a1cdb502 namespace: kserve-ci-e2e-test resourceVersion: "17584" uid: ba1e1da2-a3ae-4ca5-abce-24ec6a95b351 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d2727-predictor-6487c46c67-cphk6 namespace: kserve-ci-e2e-test resourceVersion: "17569" uid: 71cbc44b-aea9-4c53-85c7-7d03e61d0e42 kind: Event lastTimestamp: "2026-09-09T20:13:25Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:13:25Z" name: error-404-isvc-d2727-predictor-6487c46c67-cphk6.18d3c033da37db7b namespace: kserve-ci-e2e-test resourceVersion: "17608" uid: 0398c1e9-fc52-4cff-b3d2-1e34047d54d3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2727-predictor-6487c46c67-cphk6 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: 71cbc44b-aea9-4c53-85c7-7d03e61d0e42 kind: Event lastTimestamp: "2026-09-09T20:13:25Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:25Z" name: error-404-isvc-d2727-predictor-6487c46c67-cphk6.18d3c033db8fef98 namespace: kserve-ci-e2e-test resourceVersion: "17610" uid: 225181bb-3717-467b-8672-6931a0ba2b04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2727-predictor-6487c46c67-cphk6 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: 71cbc44b-aea9-4c53-85c7-7d03e61d0e42 kind: Event lastTimestamp: "2026-09-09T20:13:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:26Z" name: error-404-isvc-d2727-predictor-6487c46c67-cphk6.18d3c033de1f3ca3 namespace: kserve-ci-e2e-test resourceVersion: "17612" uid: 12b423bf-6637-41db-b80a-8b8820ed8d51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2727-predictor-6487c46c67-cphk6 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: 71cbc44b-aea9-4c53-85c7-7d03e61d0e42 kind: Event lastTimestamp: "2026-09-09T20:13:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:26Z" name: error-404-isvc-d2727-predictor-6487c46c67-cphk6.18d3c033de8ca679 namespace: kserve-ci-e2e-test resourceVersion: "17613" uid: 363b2653-d88d-4be1-8692-56ab0f40c0db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d2727-predictor-6487c46c67-cphk6 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: 71cbc44b-aea9-4c53-85c7-7d03e61d0e42 kind: Event lastTimestamp: "2026-09-09T20:13:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:26Z" name: error-404-isvc-d2727-predictor-6487c46c67-cphk6.18d3c033de963529 namespace: kserve-ci-e2e-test resourceVersion: "17614" uid: afc55ce2-8dd6-4a4f-a1a4-6d3faa03d54e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d2727-predictor-6487c46c67-cphk6 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: 71cbc44b-aea9-4c53-85c7-7d03e61d0e42 kind: Event lastTimestamp: "2026-09-09T20:13:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:26Z" name: error-404-isvc-d2727-predictor-6487c46c67-cphk6.18d3c033e18237e9 namespace: kserve-ci-e2e-test resourceVersion: "17615" uid: 45b5a64c-6b7f-4b39-abba-4ad23bfca97e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d2727-predictor-6487c46c67-cphk6 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: 71cbc44b-aea9-4c53-85c7-7d03e61d0e42 kind: Event lastTimestamp: "2026-09-09T20:13:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:26Z" name: error-404-isvc-d2727-predictor-6487c46c67-cphk6.18d3c033e1f23cc2 namespace: kserve-ci-e2e-test resourceVersion: "17616" uid: 420f7e52-5803-4e75-b217-562279372648 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:13:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2727-predictor-6487c46c67-cphk6 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: 71cbc44b-aea9-4c53-85c7-7d03e61d0e42 kind: Event lastTimestamp: "2026-09-09T20:14:02Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:13:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:02Z" name: error-404-isvc-d2727-predictor-6487c46c67-cphk6.18d3c03403fec003 namespace: kserve-ci-e2e-test resourceVersion: "17969" uid: c30aa38b-861d-4cfc-b40c-e1639b83015b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d2727-predictor-6487c46c67-cphk6 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: 71cbc44b-aea9-4c53-85c7-7d03e61d0e42 kind: Event lastTimestamp: "2026-09-09T20:22:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:39Z" name: error-404-isvc-d2727-predictor-6487c46c67-cphk6.18d3c0b4bbb6bb07 namespace: kserve-ci-e2e-test resourceVersion: "21941" uid: e019f62e-1313-4ba6-a73e-7f4f33c3b575 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d2727-predictor-6487c46c67-cphk6 namespace: kserve-ci-e2e-test resourceVersion: "17556" uid: 71cbc44b-aea9-4c53-85c7-7d03e61d0e42 kind: Event lastTimestamp: "2026-09-09T20:22:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:39Z" name: error-404-isvc-d2727-predictor-6487c46c67-cphk6.18d3c0b4bbb7c143 namespace: kserve-ci-e2e-test resourceVersion: "21944" uid: c74fc9b9-cc37-4d2c-be41-d2d8e8c2a1f5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d2727-predictor-6487c46c67 namespace: kserve-ci-e2e-test resourceVersion: "17546" uid: 7ac516a8-932f-4518-9385-4ba1b17544c9 kind: Event lastTimestamp: "2026-09-09T20:13:24Z" message: 'Created pod: error-404-isvc-d2727-predictor-6487c46c67-cphk6' metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:24Z" name: error-404-isvc-d2727-predictor-6487c46c67.18d3c0339422ab0e namespace: kserve-ci-e2e-test resourceVersion: "17555" uid: a923a9bd-7544-45b4-bf8c-999791a38ce2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d2727-predictor namespace: kserve-ci-e2e-test resourceVersion: "17544" uid: bb6dfc01-a307-4c2e-8199-70ec29d6b429 kind: Event lastTimestamp: "2026-09-09T20:13:24Z" message: Scaled up replica set error-404-isvc-d2727-predictor-6487c46c67 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:24Z" name: error-404-isvc-d2727-predictor.18d3c0339214f6df namespace: kserve-ci-e2e-test resourceVersion: "17547" uid: 0275b571-8b25-456a-ab78-98682d27967a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:13:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d2727-predictor namespace: kserve-ci-e2e-test resourceVersion: "17545" uid: b31abd28-d0ed-4606-abb7-78915ff3c4bc kind: Event lastTimestamp: "2026-09-09T20:13:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:54Z" name: error-404-isvc-d2727-predictor.18d3c037105e1daf namespace: kserve-ci-e2e-test resourceVersion: "17877" uid: 5bb4b192-3c79-448c-a674-10d16271587f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:13:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d2727-predictor namespace: kserve-ci-e2e-test resourceVersion: "17545" uid: b31abd28-d0ed-4606-abb7-78915ff3c4bc kind: Event lastTimestamp: "2026-09-09T20:13:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:54Z" name: error-404-isvc-d2727-predictor.18d3c03710c9b9c7 namespace: kserve-ci-e2e-test resourceVersion: "17878" uid: 91766118-fbed-48ac-bb74-46f7538a6df3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-09T20:14:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d2727-predictor namespace: kserve-ci-e2e-test resourceVersion: "17753" uid: b31abd28-d0ed-4606-abb7-78915ff3c4bc kind: Event lastTimestamp: "2026-09-09T20:14:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:14:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:54Z" name: error-404-isvc-d2727-predictor.18d3c03e0e22f9b9 namespace: kserve-ci-e2e-test resourceVersion: "18613" uid: f5bf9520-7d54-4d41-9385-6730bc24a606 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-09T20:14:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d2727-predictor namespace: kserve-ci-e2e-test resourceVersion: "17753" uid: b31abd28-d0ed-4606-abb7-78915ff3c4bc kind: Event lastTimestamp: "2026-09-09T20:14:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:14:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:54Z" name: error-404-isvc-d2727-predictor.18d3c03e0e881acd namespace: kserve-ci-e2e-test resourceVersion: "18614" uid: e01d1ec6-a890-4bde-81d8-559a930a9734 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:13:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d2727 namespace: kserve-ci-e2e-test resourceVersion: "17530" uid: 20d68db0-0676-4033-bc6b-3a7d565888a2 kind: Event lastTimestamp: "2026-09-09T20:14:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:14:12Z" name: error-404-isvc-d2727.18d3c03392201472 namespace: kserve-ci-e2e-test resourceVersion: "18225" uid: 99509ca1-938e-4f0c-80c9-c1b77f5dc23e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d2727 namespace: kserve-ci-e2e-test resourceVersion: "18220" uid: 20d68db0-0676-4033-bc6b-3a7d565888a2 kind: Event lastTimestamp: "2026-09-09T20:14:12Z" message: InferenceService [error-404-isvc-d2727] is Ready metadata: creationTimestamp: "2026-09-09T20:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:14:12Z" name: error-404-isvc-d2727.18d3c03ec17a8383 namespace: kserve-ci-e2e-test resourceVersion: "18221" uid: b536dc81-c306-435d-b6e9-a8aeabd428b7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:22:39.515643Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd namespace: kserve-ci-e2e-test resourceVersion: "21948" uid: 8ce218e6-084f-4bb2-adca-e1bd24b173c8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:22:39Z" name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd.18d3c0b4be4dffb5 namespace: kserve-ci-e2e-test resourceVersion: "21952" uid: 1a78e91f-1c87-4b25-be2c-a2d96c36b1b1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd namespace: kserve-ci-e2e-test resourceVersion: "21956" uid: 8ce218e6-084f-4bb2-adca-e1bd24b173c8 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:22:40Z" name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd.18d3c0b4e01128e3 namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: 415e33d1-f3b1-43cc-a4f7-08a62690cacb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: 8ce218e6-084f-4bb2-adca-e1bd24b173c8 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:40Z" name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd.18d3c0b4e161f457 namespace: kserve-ci-e2e-test resourceVersion: "21974" uid: 06067483-f554-4276-967c-3ef973b6a995 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: 8ce218e6-084f-4bb2-adca-e1bd24b173c8 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:40Z" name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd.18d3c0b4e41d48ce namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: 8e82c55f-20d1-46e2-8fe4-833eda6ffce0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: 8ce218e6-084f-4bb2-adca-e1bd24b173c8 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:40Z" name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd.18d3c0b4e48c262f namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: 6c5426ae-15e2-4b8c-b6d2-da6a20a20afc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: 8ce218e6-084f-4bb2-adca-e1bd24b173c8 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:40Z" name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd.18d3c0b4e4972e45 namespace: kserve-ci-e2e-test resourceVersion: "21977" uid: 0cd27599-910c-47f7-b77e-e9cb4e3d187d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: 8ce218e6-084f-4bb2-adca-e1bd24b173c8 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:40Z" name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd.18d3c0b4e7900642 namespace: kserve-ci-e2e-test resourceVersion: "21979" uid: 00860af3-c88d-4c40-ad40-7384bef4de45 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: 8ce218e6-084f-4bb2-adca-e1bd24b173c8 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:40Z" name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd.18d3c0b4e80399ac namespace: kserve-ci-e2e-test resourceVersion: "21980" uid: eb52c86a-7c2c-4279-bfb9-4313839fce57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: 8ce218e6-084f-4bb2-adca-e1bd24b173c8 kind: Event lastTimestamp: "2026-09-09T20:23:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:16Z" name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd.18d3c0b5102903a7 namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: 7446b77d-dc98-4c2d-8bef-585fcbc37510 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: 8ce218e6-084f-4bb2-adca-e1bd24b173c8 kind: Event lastTimestamp: "2026-09-09T20:23:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:49Z" name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd.18d3c0c5129ec801 namespace: kserve-ci-e2e-test resourceVersion: "22793" uid: 94070c05-fc80-43c9-bca9-bb8f4b2fc7cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: 8ce218e6-084f-4bb2-adca-e1bd24b173c8 kind: Event lastTimestamp: "2026-09-09T20:23:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:49Z" name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd.18d3c0c512a046ed namespace: kserve-ci-e2e-test resourceVersion: "22794" uid: 7347085a-d0e9-412f-81ac-30be16e7c47e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: 8ce218e6-084f-4bb2-adca-e1bd24b173c8 kind: Event lastTimestamp: "2026-09-09T20:23:51Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:51Z" name: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd.18d3c0c5987476b5 namespace: kserve-ci-e2e-test resourceVersion: "22893" uid: a8b905e9-3eff-4f9f-8d33-5d0ef5d36280 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e029b-predictor-6d7b6546c5 namespace: kserve-ci-e2e-test resourceVersion: "21939" uid: 0d1bf0e4-b7e7-4309-a27f-56e63254daef kind: Event lastTimestamp: "2026-09-09T20:22:39Z" message: 'Created pod: error-404-isvc-e029b-predictor-6d7b6546c5-qjfgd' metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:22:39Z" name: error-404-isvc-e029b-predictor-6d7b6546c5.18d3c0b4bdb4476e namespace: kserve-ci-e2e-test resourceVersion: "21949" uid: 2e879786-6c03-4f4a-beb2-770ee5eea2e2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e029b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21934" uid: b0b730ab-7bd1-43af-9b91-dc4e66bca2b3 kind: Event lastTimestamp: "2026-09-09T20:22:39Z" message: Scaled up replica set error-404-isvc-e029b-predictor-6d7b6546c5 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:22:39Z" name: error-404-isvc-e029b-predictor.18d3c0b4bbc46e29 namespace: kserve-ci-e2e-test resourceVersion: "21942" uid: f3b5da1f-6465-45b4-a004-c0e1e1f16073 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e029b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21938" uid: 4d3d18ae-71f6-4df5-ab5b-eda30adc3a5c kind: Event lastTimestamp: "2026-09-09T20:22:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:22:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:22:54Z" name: error-404-isvc-e029b-predictor.18d3c0b83a8b46d6 namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: 0bdb6d8f-7c7f-4af8-9112-d5606980c016 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e029b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21938" uid: 4d3d18ae-71f6-4df5-ab5b-eda30adc3a5c kind: Event lastTimestamp: "2026-09-09T20:22:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:22:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:22:54Z" name: error-404-isvc-e029b-predictor.18d3c0b83b4d6dbc namespace: kserve-ci-e2e-test resourceVersion: "22117" uid: 54f56ff9-bf5a-43eb-b186-93b756905afc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:23:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e029b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22116" uid: 4d3d18ae-71f6-4df5-ab5b-eda30adc3a5c kind: Event lastTimestamp: "2026-09-09T20:23:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:39Z" name: error-404-isvc-e029b-predictor.18d3c0bbb9fadee6 namespace: kserve-ci-e2e-test resourceVersion: "22640" uid: a85a6e5c-77bf-4237-914b-746ae3809cef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:23:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e029b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22116" uid: 4d3d18ae-71f6-4df5-ab5b-eda30adc3a5c kind: Event lastTimestamp: "2026-09-09T20:23:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:39Z" name: error-404-isvc-e029b-predictor.18d3c0bbba5d7dcb namespace: kserve-ci-e2e-test resourceVersion: "22641" uid: 0ea06912-cf41-4558-98af-092c3e9a6f57 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:22:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e029b namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: d1ca6d2f-0513-495c-a412-ef0421977676 kind: Event lastTimestamp: "2026-09-09T20:23:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:23:27Z" name: error-404-isvc-e029b.18d3c0b4bc610f12 namespace: kserve-ci-e2e-test resourceVersion: "22515" uid: 977f54f0-e0e1-42ca-b810-a3c4e3b0380d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e029b namespace: kserve-ci-e2e-test resourceVersion: "22509" uid: d1ca6d2f-0513-495c-a412-ef0421977676 kind: Event lastTimestamp: "2026-09-09T20:23:27Z" message: InferenceService [error-404-isvc-e029b] is Ready metadata: creationTimestamp: "2026-09-09T20:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:23:27Z" name: error-404-isvc-e029b.18d3c0bfcecc61f2 namespace: kserve-ci-e2e-test resourceVersion: "22511" uid: aec00662-82f2-4339-8b6b-eeba17065ab4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:11:50.390443Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16426" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-675c9cf848-w966f to ip-10-0-142-99.ec2.internal metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:11:50Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c01d9b6d89ab namespace: kserve-ci-e2e-test resourceVersion: "16431" uid: d2215ed2-f127-43a8-b872-eed461ad2a79 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:11:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:51Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c01da221fc61 namespace: kserve-ci-e2e-test resourceVersion: "16549" uid: 86d21cc4-9f77-44d1-9f80-0ce1fc391347 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16438" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:11:52Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:11:52Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c01e16fc6161 namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: 2b153899-4a8f-4a99-8e66-e2dcdf0dc3b7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:11:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5965bfb57f6e1b6e7bedc3f3d44c7470727cbff107be185731248f93ead9dac3" already present on machine metadata: creationTimestamp: "2026-09-09T20:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:52Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c01e1871d5e5 namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: fc75f13c-7114-4a53-8456-1d5caa7f9418 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:11:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-09T20:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:52Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c01e1b95c0ee namespace: kserve-ci-e2e-test resourceVersion: "16598" uid: d89ab71c-2baa-412f-a800-af4e80e788d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:11:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-09T20:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:52Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c01e1c09a265 namespace: kserve-ci-e2e-test resourceVersion: "16599" uid: 9f82d378-a724-4c7b-9d57-dd2b216d2a69 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:11:56Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907" metadata: creationTimestamp: "2026-09-09T20:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:56Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c01f023a88cc namespace: kserve-ci-e2e-test resourceVersion: "16640" uid: 23a1c4b0-93a5-4c22-82fc-6d23d1a13e7d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:12:13Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907" in 17.148s (17.148s including waiting). Image size: 1580639628 bytes.' metadata: creationTimestamp: "2026-09-09T20:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:13Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c02300619284 namespace: kserve-ci-e2e-test resourceVersion: "16815" uid: 81ebf814-59af-4921-8b99-0717c616eb17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:12:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:13Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c023035a97dc namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: 07b5dc32-f88c-4ead-8de2-6c2e61d12930 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:12:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:13Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c02303c15691 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 184421c4-32c4-4c97-84b4-7320c3d2668f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:12:13Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-09T20:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:13Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c02303cd302a namespace: kserve-ci-e2e-test resourceVersion: "16821" uid: 939c3e93-6eac-4d2a-9056-3efc546f7ddd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:12:16Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.654s (2.654s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-09T20:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:16Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c023a20bc18d namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: d86707df-c562-4410-88f8-03c4e81ab83c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:12:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:16Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c023a5b0704c namespace: kserve-ci-e2e-test resourceVersion: "16842" uid: 7d09c727-dca3-4595-ab54-6baca2519e47 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:12:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:16Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c023a61b89dd namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 7f5fd386-4bb5-40c4-97fa-863944f5f3bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-09T20:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:14:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:12Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c023b39cc912 namespace: kserve-ci-e2e-test resourceVersion: "18211" uid: a5d47019-0fe6-417c-ade8-4813c858d4e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:14:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:10Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c03e3e096fe6 namespace: kserve-ci-e2e-test resourceVersion: "18089" uid: f1b014f4-894e-497c-8557-ee23958f113f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:14:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:10Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c03e3e09d6db namespace: kserve-ci-e2e-test resourceVersion: "18090" uid: 342f4e55-ec01-432c-b7a3-a4c7f5f3e8da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: d9effee7-acc2-4a07-aeb8-1188b17afea3 kind: Event lastTimestamp: "2026-09-09T20:14:12Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:12Z" name: isvc-sklearn-graph-1-predictor-675c9cf848-w966f.18d3c03eb61bb7f8 namespace: kserve-ci-e2e-test resourceVersion: "18209" uid: 8d0e2da2-2e99-4a7a-b169-c8270053475d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-675c9cf848 namespace: kserve-ci-e2e-test resourceVersion: "16418" uid: e0d9d1d2-f93c-4615-b329-985dd49901a1 kind: Event lastTimestamp: "2026-09-09T20:11:50Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-675c9cf848-w966f' metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:11:50Z" name: isvc-sklearn-graph-1-predictor-675c9cf848.18d3c01d9afbc845 namespace: kserve-ci-e2e-test resourceVersion: "16427" uid: d00670db-3140-4e55-b300-5edbe45e2ae3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: 983e9388-49cb-4488-9d52-ca19e85a487f kind: Event lastTimestamp: "2026-09-09T20:11:50Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-675c9cf848 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:11:50Z" name: isvc-sklearn-graph-1-predictor.18d3c01d967e2b48 namespace: kserve-ci-e2e-test resourceVersion: "16419" uid: 533eed1c-d01a-4e31-9ceb-f1d1f04477dd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 132b3123-3b7e-480b-9286-cb18331b8fac kind: Event lastTimestamp: "2026-09-09T20:13:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:05Z" name: isvc-sklearn-graph-1-predictor.18d3c0211ba9370c namespace: kserve-ci-e2e-test resourceVersion: "17272" uid: 9c28a964-3649-45d5-a988-8495b6ce8a74 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16434" uid: 132b3123-3b7e-480b-9286-cb18331b8fac kind: Event lastTimestamp: "2026-09-09T20:13:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:05Z" name: isvc-sklearn-graph-1-predictor.18d3c0211c10e686 namespace: kserve-ci-e2e-test resourceVersion: "17273" uid: 55899af6-2b95-4e16-bd80-536de41946d7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-09T20:13:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16717" uid: 132b3123-3b7e-480b-9286-cb18331b8fac kind: Event lastTimestamp: "2026-09-09T20:14:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:05Z" name: isvc-sklearn-graph-1-predictor.18d3c03296cb0f53 namespace: kserve-ci-e2e-test resourceVersion: "17988" uid: bb27ccb5-507c-4c70-a10e-c1097a77bcbf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-09T20:13:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16717" uid: 132b3123-3b7e-480b-9286-cb18331b8fac kind: Event lastTimestamp: "2026-09-09T20:14:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:05Z" name: isvc-sklearn-graph-1-predictor.18d3c03297342878 namespace: kserve-ci-e2e-test resourceVersion: "17989" uid: 864229e9-89f1-4e98-90c1-76d08b850455 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: c0af476b-da74-43b6-b01b-17d85e16dfa6 kind: Event lastTimestamp: "2026-09-09T20:13:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:13:32Z" name: isvc-sklearn-graph-1.18d3c01d9c880e58 namespace: kserve-ci-e2e-test resourceVersion: "17701" uid: fd3e6c93-a6c0-4d37-a5c8-f381b5da1deb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "17699" uid: c0af476b-da74-43b6-b01b-17d85e16dfa6 kind: Event lastTimestamp: "2026-09-09T20:13:32Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-09-09T20:13:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:13:32Z" name: isvc-sklearn-graph-1.18d3c0356c71ed99 namespace: kserve-ci-e2e-test resourceVersion: "17700" uid: 603abc66-418a-4a93-adb3-50daf54d5e4e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:11:50.889062Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-667cb66696-ddx2b to ip-10-0-142-99.ec2.internal metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:11:50Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c01db925ea98 namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: c23c7ca3-c1a6-46a0-b64d-2f91206f159b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:11:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:51Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c01dc0308f0d namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: a2c85957-cc50-4656-9b87-10be589e9296 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:52Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16542" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:11:52Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:11:52Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c01dfed9209c namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: 47c14e0c-10f5-4962-8254-e902b21080ed reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:11:52Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5965bfb57f6e1b6e7bedc3f3d44c7470727cbff107be185731248f93ead9dac3" already present on machine metadata: creationTimestamp: "2026-09-09T20:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:52Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c01e00690c05 namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: b8eaa2ca-2bdb-4207-8a05-44a1487520d6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:11:52Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-09T20:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:52Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c01e03ccafaf namespace: kserve-ci-e2e-test resourceVersion: "16584" uid: 614e8e31-b5bf-45df-ac77-fbd152c01199 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:52Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:11:52Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-09T20:11:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:52Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c01e047df76d namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: 5623b82d-a7c7-4351-ad18-3e0af4779bd4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:11:56Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907" metadata: creationTimestamp: "2026-09-09T20:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:56Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c01f0225ce46 namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 887f2f2e-6a13-4351-a056-3bb2c2ee72c8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:12:13Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1907" in 17.1s (17.1s including waiting). Image size: 1580639628 bytes.' metadata: creationTimestamp: "2026-09-09T20:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:13Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c022fd66c160 namespace: kserve-ci-e2e-test resourceVersion: "16814" uid: c0f146a3-1e41-4cd6-a896-986542508474 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:12:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:13Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c023014436ed namespace: kserve-ci-e2e-test resourceVersion: "16816" uid: 0ea52674-e68d-4748-a059-9e00743d387f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:12:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:13Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c02301b09b1e namespace: kserve-ci-e2e-test resourceVersion: "16817" uid: 97c64f6d-dd68-4496-8ee9-3ab21f0cfd61 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:12:13Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-09T20:12:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:13Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c02301bfd98d namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: 41aa522b-d404-44ef-910c-4b09790c3ff2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:12:16Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.693s (2.693s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-09T20:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:16Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c023a24aeab9 namespace: kserve-ci-e2e-test resourceVersion: "16840" uid: 69872a68-95a2-4f04-be84-1839707a89ca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:12:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:16Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c023a5ac26ad namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: 51709bad-38d1-4a5d-ad9d-0e254a7f07fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:12:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:16Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c023a616609d namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: 5c507a74-456b-4542-ae69-11f303d78dee reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-09T20:12:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:13:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:12:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:22Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c023b3b89b46 namespace: kserve-ci-e2e-test resourceVersion: "17460" uid: 526b29a7-5834-4240-b9bc-1f7df69cb046 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:14:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:10Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c03e3bfb9fa9 namespace: kserve-ci-e2e-test resourceVersion: "18070" uid: 3cedcb2e-29e6-473d-9609-fc6037cbe716 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:14:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:10Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c03e3bfc57ac namespace: kserve-ci-e2e-test resourceVersion: "18078" uid: 79162d77-5e3f-4a0d-ad68-7911f4942aa9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 22da0421-7cee-4178-b8bc-0604cf2f3ba4 kind: Event lastTimestamp: "2026-09-09T20:14:12Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:12Z" name: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b.18d3c03eb61cb9ee namespace: kserve-ci-e2e-test resourceVersion: "18210" uid: 86b0de4d-bd92-4604-8948-c222fb1a3d0c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-99.ec2.internal source: component: kubelet host: ip-10-0-142-99.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-667cb66696 namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: ed1516ee-a984-428d-80b9-5fa70af591d9 kind: Event lastTimestamp: "2026-09-09T20:11:50Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-667cb66696-ddx2b' metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:11:50Z" name: isvc-sklearn-graph-2-predictor-667cb66696.18d3c01db8b0444b namespace: kserve-ci-e2e-test resourceVersion: "16531" uid: f233e889-edd9-4e9b-ad57-3bc86c79db70 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: 713e6d81-3b3d-4c3a-b117-b79c88c308a1 kind: Event lastTimestamp: "2026-09-09T20:11:50Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-667cb66696 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:11:50Z" name: isvc-sklearn-graph-2-predictor.18d3c01db61c319a namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: 684efaba-8659-468c-b551-df3509905f82 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 294cf4b2-cba7-4bb1-9436-b068531f1f2e kind: Event lastTimestamp: "2026-09-09T20:12:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:12:20Z" name: isvc-sklearn-graph-2-predictor.18d3c021347966b5 namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: 1ad4cfc2-9616-484e-a9cc-c3c30b6bd88d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16520" uid: 294cf4b2-cba7-4bb1-9436-b068531f1f2e kind: Event lastTimestamp: "2026-09-09T20:12:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:12:20Z" name: isvc-sklearn-graph-2-predictor.18d3c02134c19865 namespace: kserve-ci-e2e-test resourceVersion: "16894" uid: 7ee7cf9f-460e-48f5-aae8-e88a777375dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:12:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 294cf4b2-cba7-4bb1-9436-b068531f1f2e kind: Event lastTimestamp: "2026-09-09T20:13:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:50Z" name: isvc-sklearn-graph-2-predictor.18d3c02832296e02 namespace: kserve-ci-e2e-test resourceVersion: "17839" uid: a1c58c2e-a34b-4281-8db5-18c5843aea60 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:12:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 294cf4b2-cba7-4bb1-9436-b068531f1f2e kind: Event lastTimestamp: "2026-09-09T20:13:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:50Z" name: isvc-sklearn-graph-2-predictor.18d3c02832908459 namespace: kserve-ci-e2e-test resourceVersion: "17840" uid: cb798c9f-17c8-418b-b98c-8d842a20afac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 946a6487-8044-4025-af59-1a599c62a682 kind: Event lastTimestamp: "2026-09-09T20:13:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:13:30Z" name: isvc-sklearn-graph-2.18d3c01db5f7fcc6 namespace: kserve-ci-e2e-test resourceVersion: "17668" uid: 4b3d6269-fab9-4d8b-88ff-c815803c37ea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "17666" uid: 946a6487-8044-4025-af59-1a599c62a682 kind: Event lastTimestamp: "2026-09-09T20:13:30Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-09-09T20:13:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:13:30Z" name: isvc-sklearn-graph-2.18d3c034cf32e82d namespace: kserve-ci-e2e-test resourceVersion: "17667" uid: d82cb9ff-1486-4183-bfc5-d75ccdf9a2b5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:11:50.684053Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-4pcvk to ip-10-0-128-182.ec2.internal metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:11:50Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c01daceda24c namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: fd266b26-ffb8-4871-a191-1eed598ff3f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:11:51Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:11:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c01dcf6a3912 namespace: kserve-ci-e2e-test resourceVersion: "16564" uid: 808a9dab-4b40-4b72-8635-bf82af82b221 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:11:51Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:5965bfb57f6e1b6e7bedc3f3d44c7470727cbff107be185731248f93ead9dac3" metadata: creationTimestamp: "2026-09-09T20:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c01dd151d676 namespace: kserve-ci-e2e-test resourceVersion: "16566" uid: 07f63919-5f35-41bb-968d-74bcfd1a6237 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:11:54Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:5965bfb57f6e1b6e7bedc3f3d44c7470727cbff107be185731248f93ead9dac3" in 3.274s (3.274s including waiting). Image size: 306764700 bytes.' metadata: creationTimestamp: "2026-09-09T20:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c01e94792d6b namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: 888718cb-ad1e-4682-8a4e-835f742db3bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:11:54Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-09T20:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c01e9807f07c namespace: kserve-ci-e2e-test resourceVersion: "16619" uid: fb2d86a1-308c-4c1f-8e85-5471c1ec9ecb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:54Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:11:54Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-09T20:11:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c01e9883224b namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: 65205158-5177-4c62-ae3b-9fa35f34bf3f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:11:58Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-09T20:11:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:58Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c01f85a171db namespace: kserve-ci-e2e-test resourceVersion: "16661" uid: 04fb9be8-ee54-463f-a26f-87dd32feb556 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:12:18Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.231s (20.231s including waiting). Image size: 1332293374 bytes.' metadata: creationTimestamp: "2026-09-09T20:12:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c0243b84ec42 namespace: kserve-ci-e2e-test resourceVersion: "16869" uid: e5e0a58f-ca66-4ecd-b060-d1ad3c241102 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:12:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:12:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c0243f1801c4 namespace: kserve-ci-e2e-test resourceVersion: "16870" uid: d2051eea-d743-4b18-a508-0a2818fd36a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:12:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:12:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c0243f9237be namespace: kserve-ci-e2e-test resourceVersion: "16871" uid: 133d3654-6f58-4d77-948c-38bb40d5a63c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:12:18Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-09T20:12:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c0243fa002df namespace: kserve-ci-e2e-test resourceVersion: "16872" uid: 94c384c1-ca0b-4a06-a42f-5c02d4c6b2f1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:12:21Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.182s (2.182s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-09T20:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c024c1af98a8 namespace: kserve-ci-e2e-test resourceVersion: "16897" uid: 77b42456-1391-4de2-b173-ad7728e17119 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:12:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c024c4e980d0 namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: 765de801-213a-4762-93c9-6fe7595362b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:12:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c024c566133b namespace: kserve-ci-e2e-test resourceVersion: "16899" uid: ab81a81e-a45b-485f-963d-6e5484fc0878 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-09T20:12:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:13:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c024e80196e6 namespace: kserve-ci-e2e-test resourceVersion: "17296" uid: 324d7023-8326-4783-ae7c-1236f9e65d3e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:14:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c03e406399dd namespace: kserve-ci-e2e-test resourceVersion: "18108" uid: 2855ed72-40a2-4bea-ab62-79af644ecde4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:14:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c03e4064dbf1 namespace: kserve-ci-e2e-test resourceVersion: "18109" uid: e323ba45-c846-4ac7-a919-5b05bb63a634 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6c5b3fad-6697-4574-b6fc-7ef1aacc7af5 kind: Event lastTimestamp: "2026-09-09T20:14:12Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-4pcvk.18d3c03ec058ffa4 namespace: kserve-ci-e2e-test resourceVersion: "18218" uid: a4df6c81-4f4e-42eb-a29a-429a0e0f5106 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "16472" uid: 6e597a19-257b-4b15-991b-3c66c0b08b64 kind: Event lastTimestamp: "2026-09-09T20:11:50Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-4pcvk' metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:11:50Z" name: isvc-xgboost-graph-predictor-555fc69d56.18d3c01dac7a3757 namespace: kserve-ci-e2e-test resourceVersion: "16481" uid: c114d68c-c645-468d-8f2b-04bb226dfa47 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 558d23d9-d8bc-437e-82f2-c7bd5030b17f kind: Event lastTimestamp: "2026-09-09T20:11:50Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:11:50Z" name: isvc-xgboost-graph-predictor.18d3c01da98c2ad1 namespace: kserve-ci-e2e-test resourceVersion: "16473" uid: 27010011-4389-4590-80db-1f284a392a40 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16471" uid: 672e517a-a9d1-4a1b-bf3c-ca892561d0a4 kind: Event lastTimestamp: "2026-09-09T20:12:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:12:35Z" name: isvc-xgboost-graph-predictor.18d3c0212834a7d7 namespace: kserve-ci-e2e-test resourceVersion: "17019" uid: d1524848-d41c-4afe-afb1-7ef22fc72932 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16471" uid: 672e517a-a9d1-4a1b-bf3c-ca892561d0a4 kind: Event lastTimestamp: "2026-09-09T20:12:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:12:35Z" name: isvc-xgboost-graph-predictor.18d3c021289b86e0 namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: d1cb809c-6df3-49e1-be32-57d4578c49f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-09T20:12:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16724" uid: 672e517a-a9d1-4a1b-bf3c-ca892561d0a4 kind: Event lastTimestamp: "2026-09-09T20:13:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:12:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:35Z" name: isvc-xgboost-graph-predictor.18d3c02ba4b09c0c namespace: kserve-ci-e2e-test resourceVersion: "17721" uid: 09f8c635-6b46-43f3-ad63-db83498960ea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-09T20:12:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16724" uid: 672e517a-a9d1-4a1b-bf3c-ca892561d0a4 kind: Event lastTimestamp: "2026-09-09T20:13:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:12:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:35Z" name: isvc-xgboost-graph-predictor.18d3c02ba51c6d7f namespace: kserve-ci-e2e-test resourceVersion: "17722" uid: c6f3b8c8-619b-4894-bd4d-4e352d5323ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16455" uid: 13d63a60-f7da-4e8b-bfa0-e3b17169393d kind: Event lastTimestamp: "2026-09-09T20:13:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:13:17Z" name: isvc-xgboost-graph.18d3c01da9a8ed59 namespace: kserve-ci-e2e-test resourceVersion: "17408" uid: b3de2d1a-0736-4e2a-9428-288d25f48844 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "17405" uid: 13d63a60-f7da-4e8b-bfa0-e3b17169393d kind: Event lastTimestamp: "2026-09-09T20:13:17Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-09-09T20:13:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:13:17Z" name: isvc-xgboost-graph.18d3c031f7d2dae2 namespace: kserve-ci-e2e-test resourceVersion: "17407" uid: e5ae27ef-371c-4dd4-8680-edb261c94811 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:14:00.308106Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-55b47c64d-5br6l namespace: kserve-ci-e2e-test resourceVersion: "17925" uid: 9a58d574-b8e4-4ac7-9e78-63cba9ceb267 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-55b47c64d-5br6l to ip-10-0-128-182.ec2.internal metadata: creationTimestamp: "2026-09-09T20:14:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:14:00Z" name: model-chainer-55b47c64d-5br6l.18d3c03bdb1fc91a namespace: kserve-ci-e2e-test resourceVersion: "17930" uid: 7ae910fb-58b2-417f-8a0d-0cbe35c0cdeb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:00Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-55b47c64d-5br6l namespace: kserve-ci-e2e-test resourceVersion: "17927" uid: 9a58d574-b8e4-4ac7-9e78-63cba9ceb267 kind: Event lastTimestamp: "2026-09-09T20:14:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:00Z" name: model-chainer-55b47c64d-5br6l.18d3c03be4277173 namespace: kserve-ci-e2e-test resourceVersion: "17940" uid: d6d7b038-76d1-405a-af92-57a14d0821c0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:01Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-55b47c64d-5br6l namespace: kserve-ci-e2e-test resourceVersion: "17933" uid: 9a58d574-b8e4-4ac7-9e78-63cba9ceb267 kind: Event lastTimestamp: "2026-09-09T20:14:01Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:14:01Z" name: model-chainer-55b47c64d-5br6l.18d3c03c20e77023 namespace: kserve-ci-e2e-test resourceVersion: "17952" uid: c18343b0-d913-4321-822f-70187bfe8ca7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-55b47c64d-5br6l namespace: kserve-ci-e2e-test resourceVersion: "17927" uid: 9a58d574-b8e4-4ac7-9e78-63cba9ceb267 kind: Event lastTimestamp: "2026-09-09T20:14:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:1c50c377478387906c73b97d7029d36701f6ae847fc2e48c877fa8c0c5df33ad" already present on machine metadata: creationTimestamp: "2026-09-09T20:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:01Z" name: model-chainer-55b47c64d-5br6l.18d3c03c222e748d namespace: kserve-ci-e2e-test resourceVersion: "17954" uid: e6df31ee-306d-4a21-83ff-97bb66db38fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-55b47c64d-5br6l namespace: kserve-ci-e2e-test resourceVersion: "17927" uid: 9a58d574-b8e4-4ac7-9e78-63cba9ceb267 kind: Event lastTimestamp: "2026-09-09T20:14:01Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-09-09T20:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:01Z" name: model-chainer-55b47c64d-5br6l.18d3c03c25a808dd namespace: kserve-ci-e2e-test resourceVersion: "17955" uid: 56f5f492-0070-4591-9425-22140541fa4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-55b47c64d-5br6l namespace: kserve-ci-e2e-test resourceVersion: "17927" uid: 9a58d574-b8e4-4ac7-9e78-63cba9ceb267 kind: Event lastTimestamp: "2026-09-09T20:14:01Z" message: Started container model-chainer metadata: creationTimestamp: "2026-09-09T20:14:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:01Z" name: model-chainer-55b47c64d-5br6l.18d3c03c2618c97a namespace: kserve-ci-e2e-test resourceVersion: "17956" uid: 0a512ce8-0e9d-40e6-902d-7e07d8dcdcab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-55b47c64d-5br6l namespace: kserve-ci-e2e-test resourceVersion: "17927" uid: 9a58d574-b8e4-4ac7-9e78-63cba9ceb267 kind: Event lastTimestamp: "2026-09-09T20:14:10Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:10Z" name: model-chainer-55b47c64d-5br6l.18d3c03e35a82651 namespace: kserve-ci-e2e-test resourceVersion: "18047" uid: 826e0917-2b0e-437a-ac23-e1ef7db84ead reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:14:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-55b47c64d-5br6l namespace: kserve-ci-e2e-test resourceVersion: "17927" uid: 9a58d574-b8e4-4ac7-9e78-63cba9ceb267 kind: Event lastTimestamp: "2026-09-09T20:14:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-09T20:14:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:38Z" name: model-chainer-55b47c64d-5br6l.18d3c03edcdd08fa namespace: kserve-ci-e2e-test resourceVersion: "18478" uid: d6f7c44b-a79b-4754-8868-fdefa5476b18 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-55b47c64d namespace: kserve-ci-e2e-test resourceVersion: "17922" uid: 82070d80-1a84-4044-84cf-bd6145a99405 kind: Event lastTimestamp: "2026-09-09T20:14:00Z" message: 'Created pod: model-chainer-55b47c64d-5br6l' metadata: creationTimestamp: "2026-09-09T20:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:00Z" name: model-chainer-55b47c64d.18d3c03bdab2a098 namespace: kserve-ci-e2e-test resourceVersion: "17926" uid: 296d8cd5-f057-4e2f-b8b8-3e7d85e4f683 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17920" uid: 975971a5-8ead-46ed-b84b-dd75cea1ea49 kind: Event lastTimestamp: "2026-09-09T20:14:00Z" message: Scaled up replica set model-chainer-55b47c64d from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:00Z" name: model-chainer.18d3c03bd9eb8c64 namespace: kserve-ci-e2e-test resourceVersion: "17923" uid: a0c115ca-8cdd-4fa2-8466-120fca1db68a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17936" uid: a39f1ace-7d35-4501-9c47-054166a1de40 kind: Event lastTimestamp: "2026-09-09T20:14:00Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-09-09T20:14:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:14:00Z" name: model-chainer.18d3c03bdda9ed00 namespace: kserve-ci-e2e-test resourceVersion: "17938" uid: a1cbef95-ad27-4fe3-a66a-ae8067257cb1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:15:10.643320Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2e942-79888d85c-7dbmh namespace: kserve-ci-e2e-test resourceVersion: "18753" uid: 89bf329d-d8fc-4bb2-a3ea-11e7f36fe4e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-2e942-79888d85c-7dbmh to ip-10-0-128-182.ec2.internal metadata: creationTimestamp: "2026-09-09T20:15:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:15:10Z" name: sequence-graph-2e942-79888d85c-7dbmh.18d3c04c3b6df470 namespace: kserve-ci-e2e-test resourceVersion: "18759" uid: 75456518-0968-4293-86d5-575d4d17e791 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:15:11Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2e942-79888d85c-7dbmh namespace: kserve-ci-e2e-test resourceVersion: "18763" uid: 89bf329d-d8fc-4bb2-a3ea-11e7f36fe4e5 kind: Event lastTimestamp: "2026-09-09T20:15:11Z" message: Add eth0 [10.133.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:15:11Z" name: sequence-graph-2e942-79888d85c-7dbmh.18d3c04c5d879641 namespace: kserve-ci-e2e-test resourceVersion: "18782" uid: 53116b6a-41cd-4c23-98a8-b7d9ffd0f207 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2e942} kind: Pod name: sequence-graph-2e942-79888d85c-7dbmh namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 89bf329d-d8fc-4bb2-a3ea-11e7f36fe4e5 kind: Event lastTimestamp: "2026-09-09T20:15:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:1c50c377478387906c73b97d7029d36701f6ae847fc2e48c877fa8c0c5df33ad" already present on machine metadata: creationTimestamp: "2026-09-09T20:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:15:11Z" name: sequence-graph-2e942-79888d85c-7dbmh.18d3c04c5eadb85b namespace: kserve-ci-e2e-test resourceVersion: "18784" uid: 5d43808d-b838-444d-9844-dae94524352e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2e942} kind: Pod name: sequence-graph-2e942-79888d85c-7dbmh namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 89bf329d-d8fc-4bb2-a3ea-11e7f36fe4e5 kind: Event lastTimestamp: "2026-09-09T20:15:11Z" message: 'Created container: sequence-graph-2e942' metadata: creationTimestamp: "2026-09-09T20:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:15:11Z" name: sequence-graph-2e942-79888d85c-7dbmh.18d3c04c6188d974 namespace: kserve-ci-e2e-test resourceVersion: "18785" uid: ea48def8-5df3-4111-892a-52b3f257ab56 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:15:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2e942} kind: Pod name: sequence-graph-2e942-79888d85c-7dbmh namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 89bf329d-d8fc-4bb2-a3ea-11e7f36fe4e5 kind: Event lastTimestamp: "2026-09-09T20:15:11Z" message: Started container sequence-graph-2e942 metadata: creationTimestamp: "2026-09-09T20:15:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:15:11Z" name: sequence-graph-2e942-79888d85c-7dbmh.18d3c04c6200d983 namespace: kserve-ci-e2e-test resourceVersion: "18786" uid: 7b66c052-fc09-4e58-b7fb-4fa4055c2ab2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2e942} kind: Pod name: sequence-graph-2e942-79888d85c-7dbmh namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 89bf329d-d8fc-4bb2-a3ea-11e7f36fe4e5 kind: Event lastTimestamp: "2026-09-09T20:23:25Z" message: Stopping container sequence-graph-2e942 metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:25Z" name: sequence-graph-2e942-79888d85c-7dbmh.18d3c0bf5d78b522 namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: 999e210c-951a-4a9d-aeda-098672350ccf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2e942} kind: Pod name: sequence-graph-2e942-79888d85c-7dbmh namespace: kserve-ci-e2e-test resourceVersion: "18756" uid: 89bf329d-d8fc-4bb2-a3ea-11e7f36fe4e5 kind: Event lastTimestamp: "2026-09-09T20:23:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-09T20:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:52Z" name: sequence-graph-2e942-79888d85c-7dbmh.18d3c0bfed9ec842 namespace: kserve-ci-e2e-test resourceVersion: "22903" uid: 977dcae7-bd98-435a-a53d-8cf27156ac16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:15:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-2e942-79888d85c namespace: kserve-ci-e2e-test resourceVersion: "18750" uid: 841a0820-0212-4204-bd56-e444c2ff76e7 kind: Event lastTimestamp: "2026-09-09T20:15:10Z" message: 'Created pod: sequence-graph-2e942-79888d85c-7dbmh' metadata: creationTimestamp: "2026-09-09T20:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:15:10Z" name: sequence-graph-2e942-79888d85c.18d3c04c3af93a50 namespace: kserve-ci-e2e-test resourceVersion: "18755" uid: d362a6f4-b4e7-4243-9410-457c371d4590 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:15:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-2e942 namespace: kserve-ci-e2e-test resourceVersion: "18748" uid: 79639cbf-148a-4bb2-8444-2442b3de0c5a kind: Event lastTimestamp: "2026-09-09T20:15:10Z" message: Scaled up replica set sequence-graph-2e942-79888d85c from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:15:10Z" name: sequence-graph-2e942.18d3c04c3a0db2c2 namespace: kserve-ci-e2e-test resourceVersion: "18751" uid: 0d67cc16-2b91-4e39-84ef-79113ef49da3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:15:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-2e942 namespace: kserve-ci-e2e-test resourceVersion: "18765" uid: f66b5581-cb7d-4ba8-8ea8-5eccf64c1242 kind: Event lastTimestamp: "2026-09-09T20:15:10Z" message: InferenceGraph [sequence-graph-2e942] is Ready metadata: creationTimestamp: "2026-09-09T20:15:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:15:10Z" name: sequence-graph-2e942.18d3c04c3f905084 namespace: kserve-ci-e2e-test resourceVersion: "18770" uid: 5764b5ae-e2f4-4f86-adf2-2205fd0ea416 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:15:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2e942 namespace: kserve-ci-e2e-test resourceVersion: "18749" uid: 429349a5-fd42-4cf4-b201-fb9daba35421 kind: Event lastTimestamp: "2026-09-09T20:15:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:15:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:15:25Z" name: sequence-graph-2e942.18d3c04fb88d1af6 namespace: kserve-ci-e2e-test resourceVersion: "18895" uid: 65f8587c-57f2-43c3-9558-0209ba70490d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:15:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2e942 namespace: kserve-ci-e2e-test resourceVersion: "18749" uid: 429349a5-fd42-4cf4-b201-fb9daba35421 kind: Event lastTimestamp: "2026-09-09T20:15:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:15:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:15:25Z" name: sequence-graph-2e942.18d3c04fb8d59f1c namespace: kserve-ci-e2e-test resourceVersion: "18897" uid: f3321fe5-d09c-4996-9008-04f4a15d66e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:15:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2e942 namespace: kserve-ci-e2e-test resourceVersion: "18896" uid: 429349a5-fd42-4cf4-b201-fb9daba35421 kind: Event lastTimestamp: "2026-09-09T20:15:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:15:40Z" name: sequence-graph-2e942.18d3c053377c5539 namespace: kserve-ci-e2e-test resourceVersion: "19007" uid: bc94854a-31aa-46cf-9e73-e909728e7621 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:15:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-2e942 namespace: kserve-ci-e2e-test resourceVersion: "18896" uid: 429349a5-fd42-4cf4-b201-fb9daba35421 kind: Event lastTimestamp: "2026-09-09T20:15:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:15:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:15:40Z" name: sequence-graph-2e942.18d3c05337da6981 namespace: kserve-ci-e2e-test resourceVersion: "19008" uid: f075bffd-c71c-4eba-bc6f-1855582bc02c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:25:45.682645Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3a010-6655d97697-qtltd namespace: kserve-ci-e2e-test resourceVersion: "24150" uid: 7467bb4f-cc6c-46cf-aba6-ce41cfe86614 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-3a010-6655d97697-qtltd to ip-10-0-128-182.ec2.internal metadata: creationTimestamp: "2026-09-09T20:25:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:25:45Z" name: sequence-graph-3a010-6655d97697-qtltd.18d3c0e016b914a8 namespace: kserve-ci-e2e-test resourceVersion: "24155" uid: adaa00a3-690b-4bd8-b180-c3765c9daddd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:25:45Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3a010-6655d97697-qtltd namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: 7467bb4f-cc6c-46cf-aba6-ce41cfe86614 kind: Event lastTimestamp: "2026-09-09T20:25:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-3a010-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:25:45Z" name: sequence-graph-3a010-6655d97697-qtltd.18d3c0e022c390c2 namespace: kserve-ci-e2e-test resourceVersion: "24170" uid: 5250eda5-5c64-4ccf-9757-c9c13c6fe847 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:25:46Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3a010-6655d97697-qtltd namespace: kserve-ci-e2e-test resourceVersion: "24161" uid: 7467bb4f-cc6c-46cf-aba6-ce41cfe86614 kind: Event lastTimestamp: "2026-09-09T20:25:46Z" message: Add eth0 [10.133.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:25:46Z" name: sequence-graph-3a010-6655d97697-qtltd.18d3c0e05c8172bd namespace: kserve-ci-e2e-test resourceVersion: "24204" uid: 1a6f4072-625c-4554-b151-24bb51efe8b1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:25:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a010} kind: Pod name: sequence-graph-3a010-6655d97697-qtltd namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: 7467bb4f-cc6c-46cf-aba6-ce41cfe86614 kind: Event lastTimestamp: "2026-09-09T20:25:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:1c50c377478387906c73b97d7029d36701f6ae847fc2e48c877fa8c0c5df33ad" already present on machine metadata: creationTimestamp: "2026-09-09T20:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:25:46Z" name: sequence-graph-3a010-6655d97697-qtltd.18d3c0e05dff6c1b namespace: kserve-ci-e2e-test resourceVersion: "24206" uid: 1fa195ef-7cff-4008-b97b-aa166fe78022 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:25:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a010} kind: Pod name: sequence-graph-3a010-6655d97697-qtltd namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: 7467bb4f-cc6c-46cf-aba6-ce41cfe86614 kind: Event lastTimestamp: "2026-09-09T20:25:46Z" message: 'Created container: sequence-graph-3a010' metadata: creationTimestamp: "2026-09-09T20:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:25:46Z" name: sequence-graph-3a010-6655d97697-qtltd.18d3c0e062648a49 namespace: kserve-ci-e2e-test resourceVersion: "24209" uid: cf4ac6c8-6c4e-429c-814d-19b9379667ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:25:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a010} kind: Pod name: sequence-graph-3a010-6655d97697-qtltd namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: 7467bb4f-cc6c-46cf-aba6-ce41cfe86614 kind: Event lastTimestamp: "2026-09-09T20:25:46Z" message: Started container sequence-graph-3a010 metadata: creationTimestamp: "2026-09-09T20:25:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:25:46Z" name: sequence-graph-3a010-6655d97697-qtltd.18d3c0e062d8dae2 namespace: kserve-ci-e2e-test resourceVersion: "24210" uid: 7d3f883e-67b4-477b-8e4a-d52ab93953bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a010} kind: Pod name: sequence-graph-3a010-6655d97697-qtltd namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: 7467bb4f-cc6c-46cf-aba6-ce41cfe86614 kind: Event lastTimestamp: "2026-09-09T20:34:00Z" message: Stopping container sequence-graph-3a010 metadata: creationTimestamp: "2026-09-09T20:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:00Z" name: sequence-graph-3a010-6655d97697-qtltd.18d3c1533fb205dd namespace: kserve-ci-e2e-test resourceVersion: "27994" uid: f39a1347-c73e-41fe-978a-dfb18701f345 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:34:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a010} kind: Pod name: sequence-graph-3a010-6655d97697-qtltd namespace: kserve-ci-e2e-test resourceVersion: "24152" uid: 7467bb4f-cc6c-46cf-aba6-ce41cfe86614 kind: Event lastTimestamp: "2026-09-09T20:34:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-09T20:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:28Z" name: sequence-graph-3a010-6655d97697-qtltd.18d3c153fd3feff0 namespace: kserve-ci-e2e-test resourceVersion: "28530" uid: 1f2ab762-ee84-4537-8f7c-c9e03d79d566 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:25:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-3a010-6655d97697 namespace: kserve-ci-e2e-test resourceVersion: "24147" uid: d797fd8c-3927-4e5e-8355-3f43f5b04970 kind: Event lastTimestamp: "2026-09-09T20:25:45Z" message: 'Created pod: sequence-graph-3a010-6655d97697-qtltd' metadata: creationTimestamp: "2026-09-09T20:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:25:45Z" name: sequence-graph-3a010-6655d97697.18d3c0e0162e50a6 namespace: kserve-ci-e2e-test resourceVersion: "24153" uid: 60d627fe-6965-40b3-b779-fd6525679197 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:25:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-3a010 namespace: kserve-ci-e2e-test resourceVersion: "24145" uid: 6b5bbe9e-51d2-44e1-8279-9f1d0033db16 kind: Event lastTimestamp: "2026-09-09T20:25:45Z" message: Scaled up replica set sequence-graph-3a010-6655d97697 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:25:45Z" name: sequence-graph-3a010.18d3c0e0152cdad7 namespace: kserve-ci-e2e-test resourceVersion: "24149" uid: f88e39fb-fd85-4b6a-acf0-d4f58596dc19 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:25:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-3a010 namespace: kserve-ci-e2e-test resourceVersion: "24163" uid: f872bde2-3bb0-401a-a960-e5f5016145d8 kind: Event lastTimestamp: "2026-09-09T20:25:45Z" message: InferenceGraph [sequence-graph-3a010] is Ready metadata: creationTimestamp: "2026-09-09T20:25:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:25:45Z" name: sequence-graph-3a010.18d3c0e01a3d3275 namespace: kserve-ci-e2e-test resourceVersion: "24165" uid: bf142135-db11-420c-949c-3bb0f9f47dab reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:26:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-3a010 namespace: kserve-ci-e2e-test resourceVersion: "24146" uid: 973a49c1-1345-4f2d-bdb2-6c7a5a70048a kind: Event lastTimestamp: "2026-09-09T20:26:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:26:15Z" name: sequence-graph-3a010.18d3c0e393ea8190 namespace: kserve-ci-e2e-test resourceVersion: "24609" uid: 58f200f2-4d7e-4e71-9a33-7f9942eabb1a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:26:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-3a010 namespace: kserve-ci-e2e-test resourceVersion: "24146" uid: 973a49c1-1345-4f2d-bdb2-6c7a5a70048a kind: Event lastTimestamp: "2026-09-09T20:26:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:26:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:26:15Z" name: sequence-graph-3a010.18d3c0e3944f6113 namespace: kserve-ci-e2e-test resourceVersion: "24610" uid: 0646ae27-3c71-4bc0-971b-d5cf4632f185 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:24:35.364684Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-bf2e8-8fd9dd46d-kxhdn namespace: kserve-ci-e2e-test resourceVersion: "23285" uid: 684209fa-f5ff-47ee-a7a8-179627d5270f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-bf2e8-8fd9dd46d-kxhdn to ip-10-0-128-182.ec2.internal metadata: creationTimestamp: "2026-09-09T20:24:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:24:35Z" name: sequence-graph-bf2e8-8fd9dd46d-kxhdn.18d3c0cfb772199f namespace: kserve-ci-e2e-test resourceVersion: "23290" uid: 9258a928-9cc2-46cc-b4cb-1151d3b31948 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:35Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-bf2e8-8fd9dd46d-kxhdn namespace: kserve-ci-e2e-test resourceVersion: "23288" uid: 684209fa-f5ff-47ee-a7a8-179627d5270f kind: Event lastTimestamp: "2026-09-09T20:24:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-bf2e8-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:24:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:35Z" name: sequence-graph-bf2e8-8fd9dd46d-kxhdn.18d3c0cfbeea5ec9 namespace: kserve-ci-e2e-test resourceVersion: "23302" uid: f6022857-f47e-4dd8-9bf2-4ab80baf75a5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:36Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-bf2e8-8fd9dd46d-kxhdn namespace: kserve-ci-e2e-test resourceVersion: "23294" uid: 684209fa-f5ff-47ee-a7a8-179627d5270f kind: Event lastTimestamp: "2026-09-09T20:24:36Z" message: Add eth0 [10.133.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:24:36Z" name: sequence-graph-bf2e8-8fd9dd46d-kxhdn.18d3c0cffd80d050 namespace: kserve-ci-e2e-test resourceVersion: "23312" uid: b10d8413-7c53-4538-8b55-857b4265ce59 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-bf2e8} kind: Pod name: sequence-graph-bf2e8-8fd9dd46d-kxhdn namespace: kserve-ci-e2e-test resourceVersion: "23288" uid: 684209fa-f5ff-47ee-a7a8-179627d5270f kind: Event lastTimestamp: "2026-09-09T20:24:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:1c50c377478387906c73b97d7029d36701f6ae847fc2e48c877fa8c0c5df33ad" already present on machine metadata: creationTimestamp: "2026-09-09T20:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:36Z" name: sequence-graph-bf2e8-8fd9dd46d-kxhdn.18d3c0cffefab78e namespace: kserve-ci-e2e-test resourceVersion: "23314" uid: 5cf690c1-7de9-45c6-aac9-c8dd9eee7231 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-bf2e8} kind: Pod name: sequence-graph-bf2e8-8fd9dd46d-kxhdn namespace: kserve-ci-e2e-test resourceVersion: "23288" uid: 684209fa-f5ff-47ee-a7a8-179627d5270f kind: Event lastTimestamp: "2026-09-09T20:24:36Z" message: 'Created container: sequence-graph-bf2e8' metadata: creationTimestamp: "2026-09-09T20:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:36Z" name: sequence-graph-bf2e8-8fd9dd46d-kxhdn.18d3c0d00248a6f2 namespace: kserve-ci-e2e-test resourceVersion: "23315" uid: 421db72a-4400-4e79-9f29-aed8450e851e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-bf2e8} kind: Pod name: sequence-graph-bf2e8-8fd9dd46d-kxhdn namespace: kserve-ci-e2e-test resourceVersion: "23288" uid: 684209fa-f5ff-47ee-a7a8-179627d5270f kind: Event lastTimestamp: "2026-09-09T20:24:36Z" message: Started container sequence-graph-bf2e8 metadata: creationTimestamp: "2026-09-09T20:24:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:36Z" name: sequence-graph-bf2e8-8fd9dd46d-kxhdn.18d3c0d002d2fbdc namespace: kserve-ci-e2e-test resourceVersion: "23316" uid: cde98514-61de-468e-9db1-4750787ceff3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-bf2e8} kind: Pod name: sequence-graph-bf2e8-8fd9dd46d-kxhdn namespace: kserve-ci-e2e-test resourceVersion: "23288" uid: 684209fa-f5ff-47ee-a7a8-179627d5270f kind: Event lastTimestamp: "2026-09-09T20:24:45Z" message: Stopping container sequence-graph-bf2e8 metadata: creationTimestamp: "2026-09-09T20:24:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:45Z" name: sequence-graph-bf2e8-8fd9dd46d-kxhdn.18d3c0d20ee2fa7e namespace: kserve-ci-e2e-test resourceVersion: "23425" uid: 474a5f3c-7ef0-45e4-a80b-0b735fb64132 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:24:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-bf2e8} kind: Pod name: sequence-graph-bf2e8-8fd9dd46d-kxhdn namespace: kserve-ci-e2e-test resourceVersion: "23288" uid: 684209fa-f5ff-47ee-a7a8-179627d5270f kind: Event lastTimestamp: "2026-09-09T20:25:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-09T20:24:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:25:13Z" name: sequence-graph-bf2e8-8fd9dd46d-kxhdn.18d3c0d2b09867fc namespace: kserve-ci-e2e-test resourceVersion: "23813" uid: be1d4d0f-ef26-417e-8b53-8c188e4ff662 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-bf2e8-8fd9dd46d namespace: kserve-ci-e2e-test resourceVersion: "23282" uid: b48a9cfd-22e2-48fd-93b0-42ad9e654195 kind: Event lastTimestamp: "2026-09-09T20:24:35Z" message: 'Created pod: sequence-graph-bf2e8-8fd9dd46d-kxhdn' metadata: creationTimestamp: "2026-09-09T20:24:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:35Z" name: sequence-graph-bf2e8-8fd9dd46d.18d3c0cfb70d0740 namespace: kserve-ci-e2e-test resourceVersion: "23287" uid: 55afd484-ccb6-4c70-8db0-b54c0ab53412 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-bf2e8 namespace: kserve-ci-e2e-test resourceVersion: "23280" uid: d3ee7755-deec-4e54-aeb5-b5a56ecc6102 kind: Event lastTimestamp: "2026-09-09T20:24:35Z" message: Scaled up replica set sequence-graph-bf2e8-8fd9dd46d from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:24:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:35Z" name: sequence-graph-bf2e8.18d3c0cfb614fee3 namespace: kserve-ci-e2e-test resourceVersion: "23283" uid: d3ad1a45-0233-43c9-b8f2-25f10b663c54 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-bf2e8 namespace: kserve-ci-e2e-test resourceVersion: "23296" uid: 4c0ad375-1403-471f-b3ca-5852a0744398 kind: Event lastTimestamp: "2026-09-09T20:24:35Z" message: InferenceGraph [sequence-graph-bf2e8] is Ready metadata: creationTimestamp: "2026-09-09T20:24:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:24:35Z" name: sequence-graph-bf2e8.18d3c0cfbbc6ab23 namespace: kserve-ci-e2e-test resourceVersion: "23298" uid: 29fafe31-ddef-43ae-a4d6-73b6fd3da989 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:34:04.601665Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-97f7b-77f98465f9-tzcgp namespace: kserve-ci-e2e-test resourceVersion: "28160" uid: 8eabfcaf-1d1c-49b5-ae38-fa1e78e216fe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-97f7b-77f98465f9-tzcgp to ip-10-0-128-182.ec2.internal metadata: creationTimestamp: "2026-09-09T20:34:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:34:04Z" name: splitter-graph-97f7b-77f98465f9-tzcgp.18d3c154409d1a4e namespace: kserve-ci-e2e-test resourceVersion: "28165" uid: a2f52e25-8f27-4640-af52-24397252a2cc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:05Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-97f7b-77f98465f9-tzcgp namespace: kserve-ci-e2e-test resourceVersion: "28169" uid: 8eabfcaf-1d1c-49b5-ae38-fa1e78e216fe kind: Event lastTimestamp: "2026-09-09T20:34:05Z" message: Add eth0 [10.133.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:34:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:34:05Z" name: splitter-graph-97f7b-77f98465f9-tzcgp.18d3c15462b158a6 namespace: kserve-ci-e2e-test resourceVersion: "28188" uid: 47d7297e-d30e-4af5-afed-1942ca3c1a2d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-97f7b} kind: Pod name: splitter-graph-97f7b-77f98465f9-tzcgp namespace: kserve-ci-e2e-test resourceVersion: "28161" uid: 8eabfcaf-1d1c-49b5-ae38-fa1e78e216fe kind: Event lastTimestamp: "2026-09-09T20:34:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:1c50c377478387906c73b97d7029d36701f6ae847fc2e48c877fa8c0c5df33ad" already present on machine metadata: creationTimestamp: "2026-09-09T20:34:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:05Z" name: splitter-graph-97f7b-77f98465f9-tzcgp.18d3c154642f0273 namespace: kserve-ci-e2e-test resourceVersion: "28190" uid: b7f91188-baf8-45d4-ae67-cb145e716a40 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-97f7b} kind: Pod name: splitter-graph-97f7b-77f98465f9-tzcgp namespace: kserve-ci-e2e-test resourceVersion: "28161" uid: 8eabfcaf-1d1c-49b5-ae38-fa1e78e216fe kind: Event lastTimestamp: "2026-09-09T20:34:05Z" message: 'Created container: splitter-graph-97f7b' metadata: creationTimestamp: "2026-09-09T20:34:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:05Z" name: splitter-graph-97f7b-77f98465f9-tzcgp.18d3c154677dd7e6 namespace: kserve-ci-e2e-test resourceVersion: "28191" uid: a509c5fe-14ef-4687-88f3-060b463a8f87 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-97f7b} kind: Pod name: splitter-graph-97f7b-77f98465f9-tzcgp namespace: kserve-ci-e2e-test resourceVersion: "28161" uid: 8eabfcaf-1d1c-49b5-ae38-fa1e78e216fe kind: Event lastTimestamp: "2026-09-09T20:34:05Z" message: Started container splitter-graph-97f7b metadata: creationTimestamp: "2026-09-09T20:34:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:05Z" name: splitter-graph-97f7b-77f98465f9-tzcgp.18d3c15467f23958 namespace: kserve-ci-e2e-test resourceVersion: "28192" uid: 8e9cd87e-164a-43b2-98e2-b3d85848d038 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-97f7b} kind: Pod name: splitter-graph-97f7b-77f98465f9-tzcgp namespace: kserve-ci-e2e-test resourceVersion: "28161" uid: 8eabfcaf-1d1c-49b5-ae38-fa1e78e216fe kind: Event lastTimestamp: "2026-09-09T20:34:14Z" message: Stopping container splitter-graph-97f7b metadata: creationTimestamp: "2026-09-09T20:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:14Z" name: splitter-graph-97f7b-77f98465f9-tzcgp.18d3c15699c1c97e namespace: kserve-ci-e2e-test resourceVersion: "28286" uid: 1e95f87c-bcba-46c9-9888-fe46bf112ee5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:34:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-97f7b} kind: Pod name: splitter-graph-97f7b-77f98465f9-tzcgp namespace: kserve-ci-e2e-test resourceVersion: "28161" uid: 8eabfcaf-1d1c-49b5-ae38-fa1e78e216fe kind: Event lastTimestamp: "2026-09-09T20:34:41Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-09T20:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:41Z" name: splitter-graph-97f7b-77f98465f9-tzcgp.18d3c15716fe7199 namespace: kserve-ci-e2e-test resourceVersion: "28639" uid: fd9d2d17-7c17-4be8-b9db-462cd5bfb355 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-97f7b-77f98465f9 namespace: kserve-ci-e2e-test resourceVersion: "28156" uid: c09d4a8b-f540-47b8-87eb-1d827db70b01 kind: Event lastTimestamp: "2026-09-09T20:34:04Z" message: 'Created pod: splitter-graph-97f7b-77f98465f9-tzcgp' metadata: creationTimestamp: "2026-09-09T20:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:04Z" name: splitter-graph-97f7b-77f98465f9.18d3c15440395a1c namespace: kserve-ci-e2e-test resourceVersion: "28162" uid: 4686aed6-2113-496b-bfe8-d2f6b8b9b252 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-97f7b namespace: kserve-ci-e2e-test resourceVersion: "28153" uid: 36bc8d86-e4e0-4795-afb0-4cbc19e78ec3 kind: Event lastTimestamp: "2026-09-09T20:34:04Z" message: Scaled up replica set splitter-graph-97f7b-77f98465f9 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:04Z" name: splitter-graph-97f7b.18d3c1543f86b921 namespace: kserve-ci-e2e-test resourceVersion: "28158" uid: 71620496-3020-42b8-b6c3-1901560cc06d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:04Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-97f7b namespace: kserve-ci-e2e-test resourceVersion: "28172" uid: dba3f1da-b8f7-4a08-8dcc-ba94b1d94886 kind: Event lastTimestamp: "2026-09-09T20:34:04Z" message: InferenceGraph [splitter-graph-97f7b] is Ready metadata: creationTimestamp: "2026-09-09T20:34:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:34:04Z" name: splitter-graph-97f7b.18d3c1544520c907 namespace: kserve-ci-e2e-test resourceVersion: "28174" uid: 0647c808-611c-492b-b8fc-6b8b33c01f52 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:35:24.906507Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b9ccb-766d4cf7d6-rd6m7 namespace: kserve-ci-e2e-test resourceVersion: "29056" uid: a253f913-ce35-48f8-b081-0f5efb64c821 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-b9ccb-766d4cf7d6-rd6m7 to ip-10-0-128-182.ec2.internal metadata: creationTimestamp: "2026-09-09T20:35:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:35:24Z" name: splitter-graph-b9ccb-766d4cf7d6-rd6m7.18d3c166f327c315 namespace: kserve-ci-e2e-test resourceVersion: "29063" uid: 1c701fd5-81fb-4a07-981f-fa82f3f29320 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:25Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b9ccb-766d4cf7d6-rd6m7 namespace: kserve-ci-e2e-test resourceVersion: "29059" uid: a253f913-ce35-48f8-b081-0f5efb64c821 kind: Event lastTimestamp: "2026-09-09T20:35:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-b9ccb-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:35:25Z" name: splitter-graph-b9ccb-766d4cf7d6-rd6m7.18d3c166fe328b7c namespace: kserve-ci-e2e-test resourceVersion: "29075" uid: 3f2388b5-f31d-4d28-8c06-8104beb491fd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:26Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b9ccb-766d4cf7d6-rd6m7 namespace: kserve-ci-e2e-test resourceVersion: "29066" uid: a253f913-ce35-48f8-b081-0f5efb64c821 kind: Event lastTimestamp: "2026-09-09T20:35:26Z" message: Add eth0 [10.133.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:35:26Z" name: splitter-graph-b9ccb-766d4cf7d6-rd6m7.18d3c167391ed2ce namespace: kserve-ci-e2e-test resourceVersion: "29085" uid: c0b43aec-5af0-4abf-beb9-3a0f5a72e7c8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b9ccb} kind: Pod name: splitter-graph-b9ccb-766d4cf7d6-rd6m7 namespace: kserve-ci-e2e-test resourceVersion: "29059" uid: a253f913-ce35-48f8-b081-0f5efb64c821 kind: Event lastTimestamp: "2026-09-09T20:35:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:1c50c377478387906c73b97d7029d36701f6ae847fc2e48c877fa8c0c5df33ad" already present on machine metadata: creationTimestamp: "2026-09-09T20:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:35:26Z" name: splitter-graph-b9ccb-766d4cf7d6-rd6m7.18d3c1673a6d0d11 namespace: kserve-ci-e2e-test resourceVersion: "29087" uid: faa40e1e-11e9-4ddf-85eb-7c8783f576ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b9ccb} kind: Pod name: splitter-graph-b9ccb-766d4cf7d6-rd6m7 namespace: kserve-ci-e2e-test resourceVersion: "29059" uid: a253f913-ce35-48f8-b081-0f5efb64c821 kind: Event lastTimestamp: "2026-09-09T20:35:26Z" message: 'Created container: splitter-graph-b9ccb' metadata: creationTimestamp: "2026-09-09T20:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:35:26Z" name: splitter-graph-b9ccb-766d4cf7d6-rd6m7.18d3c1673dabd85b namespace: kserve-ci-e2e-test resourceVersion: "29088" uid: e3de2aa1-39b2-44dd-a8b0-4410223640ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b9ccb} kind: Pod name: splitter-graph-b9ccb-766d4cf7d6-rd6m7 namespace: kserve-ci-e2e-test resourceVersion: "29059" uid: a253f913-ce35-48f8-b081-0f5efb64c821 kind: Event lastTimestamp: "2026-09-09T20:35:26Z" message: Started container splitter-graph-b9ccb metadata: creationTimestamp: "2026-09-09T20:35:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:35:26Z" name: splitter-graph-b9ccb-766d4cf7d6-rd6m7.18d3c1673e2454b9 namespace: kserve-ci-e2e-test resourceVersion: "29089" uid: f59b0605-e106-44ec-a4cd-bca12dee1885 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b9ccb} kind: Pod name: splitter-graph-b9ccb-766d4cf7d6-rd6m7 namespace: kserve-ci-e2e-test resourceVersion: "29059" uid: a253f913-ce35-48f8-b081-0f5efb64c821 kind: Event lastTimestamp: "2026-09-09T20:43:39Z" message: Stopping container splitter-graph-b9ccb metadata: creationTimestamp: "2026-09-09T20:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:43:39Z" name: splitter-graph-b9ccb-766d4cf7d6-rd6m7.18d3c1da1f9574e8 namespace: kserve-ci-e2e-test resourceVersion: "32535" uid: ee11b017-cef6-4cd1-abb1-93e357f7dd13 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b9ccb} kind: Pod name: splitter-graph-b9ccb-766d4cf7d6-rd6m7 namespace: kserve-ci-e2e-test resourceVersion: "29059" uid: a253f913-ce35-48f8-b081-0f5efb64c821 kind: Event lastTimestamp: "2026-09-09T20:44:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-09T20:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:44:08Z" name: splitter-graph-b9ccb-766d4cf7d6-rd6m7.18d3c1daf5e5570f namespace: kserve-ci-e2e-test resourceVersion: "32777" uid: 233314c1-2fc0-4f93-91bf-cbb446f97c32 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-b9ccb-766d4cf7d6 namespace: kserve-ci-e2e-test resourceVersion: "29053" uid: b17e1ab8-1ddf-45be-8289-381bbfe2b7c0 kind: Event lastTimestamp: "2026-09-09T20:35:24Z" message: 'Created pod: splitter-graph-b9ccb-766d4cf7d6-rd6m7' metadata: creationTimestamp: "2026-09-09T20:35:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:24Z" name: splitter-graph-b9ccb-766d4cf7d6.18d3c166f2be5326 namespace: kserve-ci-e2e-test resourceVersion: "29058" uid: e353cbf2-cad2-493e-8fe8-140a81a8ead7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-b9ccb namespace: kserve-ci-e2e-test resourceVersion: "29051" uid: 083a3ee0-eef2-40ce-98da-e8aef35811d3 kind: Event lastTimestamp: "2026-09-09T20:35:24Z" message: Scaled up replica set splitter-graph-b9ccb-766d4cf7d6 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:35:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:24Z" name: splitter-graph-b9ccb.18d3c166f21e74ef namespace: kserve-ci-e2e-test resourceVersion: "29054" uid: ac9d3042-4ae5-4895-b374-31a6b714ee59 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-b9ccb namespace: kserve-ci-e2e-test resourceVersion: "29069" uid: 6d6ea80c-cc8b-4775-9224-89bc014c2b93 kind: Event lastTimestamp: "2026-09-09T20:35:24Z" message: InferenceGraph [splitter-graph-b9ccb] is Ready metadata: creationTimestamp: "2026-09-09T20:35:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:35:24Z" name: splitter-graph-b9ccb.18d3c166f76e90fd namespace: kserve-ci-e2e-test resourceVersion: "29071" uid: 1b510efe-780f-474c-976c-c32a6a78ad73 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-b9ccb namespace: kserve-ci-e2e-test resourceVersion: "29052" uid: 5f9536a7-397c-4078-bdd2-ce5eeae75abf kind: Event lastTimestamp: "2026-09-09T20:35:39Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:35:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:39Z" name: splitter-graph-b9ccb.18d3c16a71016cee namespace: kserve-ci-e2e-test resourceVersion: "29209" uid: 4cdccb15-3633-44e4-9f84-4305d09173a3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-b9ccb namespace: kserve-ci-e2e-test resourceVersion: "29052" uid: 5f9536a7-397c-4078-bdd2-ce5eeae75abf kind: Event lastTimestamp: "2026-09-09T20:35:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:35:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:39Z" name: splitter-graph-b9ccb.18d3c16a716df6ca namespace: kserve-ci-e2e-test resourceVersion: "29211" uid: bade06f8-bea4-4267-8ba5-d978e5a5af71 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-b9ccb namespace: kserve-ci-e2e-test resourceVersion: "29210" uid: 5f9536a7-397c-4078-bdd2-ce5eeae75abf kind: Event lastTimestamp: "2026-09-09T20:35:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:35:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:54Z" name: splitter-graph-b9ccb.18d3c16df048fe8e namespace: kserve-ci-e2e-test resourceVersion: "29320" uid: 3ae823ef-955c-47f1-8751-767d6f8ed410 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-b9ccb namespace: kserve-ci-e2e-test resourceVersion: "29210" uid: 5f9536a7-397c-4078-bdd2-ce5eeae75abf kind: Event lastTimestamp: "2026-09-09T20:35:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:35:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:54Z" name: splitter-graph-b9ccb.18d3c16df09b8582 namespace: kserve-ci-e2e-test resourceVersion: "29321" uid: ad295fde-364d-43bf-87b5-e02cc58e2e7a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:14:10.692632Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr namespace: kserve-ci-e2e-test resourceVersion: "18125" uid: 4be82fbf-6585-47da-abbb-c8ef329043ec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2e942-predictor-6569cb6498-x7nfr to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:14:10Z" name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr.18d3c03e46170b58 namespace: kserve-ci-e2e-test resourceVersion: "18130" uid: b80515a9-c72a-4bc6-8c5b-14ad7ff599a0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr namespace: kserve-ci-e2e-test resourceVersion: "18127" uid: 4be82fbf-6585-47da-abbb-c8ef329043ec kind: Event lastTimestamp: "2026-09-09T20:14:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-2e942-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:10Z" name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr.18d3c03e4f07ad30 namespace: kserve-ci-e2e-test resourceVersion: "18160" uid: 55578568-e54d-45fa-8d4f-38cdcf69ada1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr namespace: kserve-ci-e2e-test resourceVersion: "18135" uid: 4be82fbf-6585-47da-abbb-c8ef329043ec kind: Event lastTimestamp: "2026-09-09T20:14:11Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:14:11Z" name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr.18d3c03e8bde9d97 namespace: kserve-ci-e2e-test resourceVersion: "18186" uid: a3f3422d-019c-4674-bc9c-59684947d283 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr namespace: kserve-ci-e2e-test resourceVersion: "18127" uid: 4be82fbf-6585-47da-abbb-c8ef329043ec kind: Event lastTimestamp: "2026-09-09T20:14:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:11Z" name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr.18d3c03e8d851231 namespace: kserve-ci-e2e-test resourceVersion: "18189" uid: 19c42432-d210-4e1c-a2c7-03016bd4f566 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr namespace: kserve-ci-e2e-test resourceVersion: "18127" uid: 4be82fbf-6585-47da-abbb-c8ef329043ec kind: Event lastTimestamp: "2026-09-09T20:14:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:11Z" name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr.18d3c03e908a4f50 namespace: kserve-ci-e2e-test resourceVersion: "18190" uid: 6c389b48-81fc-4830-b116-bd7187084a32 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr namespace: kserve-ci-e2e-test resourceVersion: "18127" uid: 4be82fbf-6585-47da-abbb-c8ef329043ec kind: Event lastTimestamp: "2026-09-09T20:14:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:11Z" name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr.18d3c03e90fc6abe namespace: kserve-ci-e2e-test resourceVersion: "18191" uid: a3fabb40-7033-4800-8efb-ef24f30df5f4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr namespace: kserve-ci-e2e-test resourceVersion: "18127" uid: 4be82fbf-6585-47da-abbb-c8ef329043ec kind: Event lastTimestamp: "2026-09-09T20:14:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:11Z" name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr.18d3c03e91070917 namespace: kserve-ci-e2e-test resourceVersion: "18193" uid: 07a5a0a3-317a-4c7f-8391-00451c04dc8f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr namespace: kserve-ci-e2e-test resourceVersion: "18127" uid: 4be82fbf-6585-47da-abbb-c8ef329043ec kind: Event lastTimestamp: "2026-09-09T20:14:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:12Z" name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr.18d3c03e951ec531 namespace: kserve-ci-e2e-test resourceVersion: "18197" uid: dd4a42ef-6cf6-49d2-ba43-c84b8fa6ba90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr namespace: kserve-ci-e2e-test resourceVersion: "18127" uid: 4be82fbf-6585-47da-abbb-c8ef329043ec kind: Event lastTimestamp: "2026-09-09T20:14:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:12Z" name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr.18d3c03e959618df namespace: kserve-ci-e2e-test resourceVersion: "18198" uid: 9a55d26e-2524-4959-9bee-03ed30fc7382 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:14:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr namespace: kserve-ci-e2e-test resourceVersion: "18127" uid: 4be82fbf-6585-47da-abbb-c8ef329043ec kind: Event lastTimestamp: "2026-09-09T20:14:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:14:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:48Z" name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr.18d3c03ec4f60e03 namespace: kserve-ci-e2e-test resourceVersion: "18572" uid: 4b144d54-37fa-430f-89b9-7265dda42cd3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr namespace: kserve-ci-e2e-test resourceVersion: "18127" uid: 4be82fbf-6585-47da-abbb-c8ef329043ec kind: Event lastTimestamp: "2026-09-09T20:23:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:25Z" name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr.18d3c0bf63933eae namespace: kserve-ci-e2e-test resourceVersion: "22382" uid: 8c87db43-6e60-43ce-b0e8-bd1738f76de6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr namespace: kserve-ci-e2e-test resourceVersion: "18127" uid: 4be82fbf-6585-47da-abbb-c8ef329043ec kind: Event lastTimestamp: "2026-09-09T20:23:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:25Z" name: success-200-isvc-2e942-predictor-6569cb6498-x7nfr.18d3c0bf6393db0e namespace: kserve-ci-e2e-test resourceVersion: "22383" uid: 4234411e-915b-4a50-a942-2634c8c59d04 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2e942-predictor-6569cb6498 namespace: kserve-ci-e2e-test resourceVersion: "18119" uid: dfddd431-bc77-4fc5-80be-f9dc286b5336 kind: Event lastTimestamp: "2026-09-09T20:14:10Z" message: 'Created pod: success-200-isvc-2e942-predictor-6569cb6498-x7nfr' metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:10Z" name: success-200-isvc-2e942-predictor-6569cb6498.18d3c03e45a59106 namespace: kserve-ci-e2e-test resourceVersion: "18126" uid: 82163c64-8b4c-417c-8b9a-f5880c5a6a7d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2e942-predictor namespace: kserve-ci-e2e-test resourceVersion: "18117" uid: 5a2e092e-4039-48f7-887e-7e9e1c36f009 kind: Event lastTimestamp: "2026-09-09T20:14:10Z" message: Scaled up replica set success-200-isvc-2e942-predictor-6569cb6498 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:10Z" name: success-200-isvc-2e942-predictor.18d3c03e43b69c7f namespace: kserve-ci-e2e-test resourceVersion: "18120" uid: 03a757d4-d373-4939-87d7-52ca2d455ec1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:14:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2e942-predictor namespace: kserve-ci-e2e-test resourceVersion: "18118" uid: 1414f077-97cd-4556-b312-d96e768bdde6 kind: Event lastTimestamp: "2026-09-09T20:14:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:14:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:55Z" name: success-200-isvc-2e942-predictor.18d3c041c246dfa0 namespace: kserve-ci-e2e-test resourceVersion: "18622" uid: 361905a0-5839-4107-8ad5-e5a43a60bdd8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:14:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2e942-predictor namespace: kserve-ci-e2e-test resourceVersion: "18118" uid: 1414f077-97cd-4556-b312-d96e768bdde6 kind: Event lastTimestamp: "2026-09-09T20:14:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:14:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:55Z" name: success-200-isvc-2e942-predictor.18d3c041c2a7e819 namespace: kserve-ci-e2e-test resourceVersion: "18623" uid: a1da098a-a6c6-4e0e-b690-0fa8d179d17d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:14:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2e942-predictor namespace: kserve-ci-e2e-test resourceVersion: "18373" uid: 1414f077-97cd-4556-b312-d96e768bdde6 kind: Event lastTimestamp: "2026-09-09T20:15:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:14:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:15:25Z" name: success-200-isvc-2e942-predictor.18d3c045414a1ecc namespace: kserve-ci-e2e-test resourceVersion: "18898" uid: 51c4c865-302e-4c91-9e26-d0e131f8634e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:14:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2e942-predictor namespace: kserve-ci-e2e-test resourceVersion: "18373" uid: 1414f077-97cd-4556-b312-d96e768bdde6 kind: Event lastTimestamp: "2026-09-09T20:15:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:14:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:15:25Z" name: success-200-isvc-2e942-predictor.18d3c04541a930c3 namespace: kserve-ci-e2e-test resourceVersion: "18899" uid: aabf29e0-3296-448f-b609-73e420abfc88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:14:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2e942 namespace: kserve-ci-e2e-test resourceVersion: "18105" uid: 577eaed8-43e6-4c17-9474-c5e02d514934 kind: Event lastTimestamp: "2026-09-09T20:14:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:14:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:14:59Z" name: success-200-isvc-2e942.18d3c03e43cc62e6 namespace: kserve-ci-e2e-test resourceVersion: "18664" uid: 05951403-4282-4cdf-b43e-1cfd096e97f4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2e942 namespace: kserve-ci-e2e-test resourceVersion: "18659" uid: 577eaed8-43e6-4c17-9474-c5e02d514934 kind: Event lastTimestamp: "2026-09-09T20:14:58Z" message: InferenceService [success-200-isvc-2e942] is Ready metadata: creationTimestamp: "2026-09-09T20:14:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:14:58Z" name: success-200-isvc-2e942.18d3c04983b47ed6 namespace: kserve-ci-e2e-test resourceVersion: "18661" uid: 39b3be0d-a813-44bb-94f8-7229d4513e85 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:24:45.651354Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2 namespace: kserve-ci-e2e-test resourceVersion: "23480" uid: e92c8cc4-69f1-42cc-bfb5-39c7ba89de01 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2 to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:24:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:24:45Z" name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2.18d3c0d21c944829 namespace: kserve-ci-e2e-test resourceVersion: "23485" uid: b999548f-3a6d-45bd-986d-4678f9f80345 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2 namespace: kserve-ci-e2e-test resourceVersion: "23489" uid: e92c8cc4-69f1-42cc-bfb5-39c7ba89de01 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:24:46Z" name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2.18d3c0d23e661ad0 namespace: kserve-ci-e2e-test resourceVersion: "23527" uid: 90f4fb4e-b079-48fc-9e60-480ff622ba04 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2 namespace: kserve-ci-e2e-test resourceVersion: "23482" uid: e92c8cc4-69f1-42cc-bfb5-39c7ba89de01 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:46Z" name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2.18d3c0d23fa10174 namespace: kserve-ci-e2e-test resourceVersion: "23530" uid: 6d0f9b24-42a5-45c2-9cd0-9be29b058d40 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2 namespace: kserve-ci-e2e-test resourceVersion: "23482" uid: e92c8cc4-69f1-42cc-bfb5-39c7ba89de01 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:46Z" name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2.18d3c0d242560d4a namespace: kserve-ci-e2e-test resourceVersion: "23531" uid: 7fbfb50a-24bd-4ecc-91e2-d5ec5555ec76 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2 namespace: kserve-ci-e2e-test resourceVersion: "23482" uid: e92c8cc4-69f1-42cc-bfb5-39c7ba89de01 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:46Z" name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2.18d3c0d242c3c150 namespace: kserve-ci-e2e-test resourceVersion: "23532" uid: 21f21c54-be42-4bd1-9891-cd37940d4c35 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2 namespace: kserve-ci-e2e-test resourceVersion: "23482" uid: e92c8cc4-69f1-42cc-bfb5-39c7ba89de01 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:46Z" name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2.18d3c0d242cdfac3 namespace: kserve-ci-e2e-test resourceVersion: "23533" uid: f582e3bb-b17c-438c-9424-8893cdea8967 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2 namespace: kserve-ci-e2e-test resourceVersion: "23482" uid: e92c8cc4-69f1-42cc-bfb5-39c7ba89de01 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:46Z" name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2.18d3c0d2471096d6 namespace: kserve-ci-e2e-test resourceVersion: "23537" uid: ad1d9371-b1b2-4b14-9dea-0f54876e4cb7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2 namespace: kserve-ci-e2e-test resourceVersion: "23482" uid: e92c8cc4-69f1-42cc-bfb5-39c7ba89de01 kind: Event lastTimestamp: "2026-09-09T20:24:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:46Z" name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2.18d3c0d247874d84 namespace: kserve-ci-e2e-test resourceVersion: "23538" uid: fffd1682-1196-46d5-896b-e6a95bc3803e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-09T20:24:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2 namespace: kserve-ci-e2e-test resourceVersion: "23482" uid: e92c8cc4-69f1-42cc-bfb5-39c7ba89de01 kind: Event lastTimestamp: "2026-09-09T20:34:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:24:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:03Z" name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2.18d3c0d24c1fa4b2 namespace: kserve-ci-e2e-test resourceVersion: "28136" uid: b56e6a87-ca11-46b8-8b5c-c66e660314c5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2 namespace: kserve-ci-e2e-test resourceVersion: "23482" uid: e92c8cc4-69f1-42cc-bfb5-39c7ba89de01 kind: Event lastTimestamp: "2026-09-09T20:34:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:00Z" name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2.18d3c15345468668 namespace: kserve-ci-e2e-test resourceVersion: "28012" uid: 236042e6-c16c-4347-818b-cea99518fd3a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2 namespace: kserve-ci-e2e-test resourceVersion: "23482" uid: e92c8cc4-69f1-42cc-bfb5-39c7ba89de01 kind: Event lastTimestamp: "2026-09-09T20:34:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:00Z" name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2.18d3c1534547d679 namespace: kserve-ci-e2e-test resourceVersion: "28014" uid: 04ea32e2-687c-4c07-a694-08456a084d65 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2 namespace: kserve-ci-e2e-test resourceVersion: "23482" uid: e92c8cc4-69f1-42cc-bfb5-39c7ba89de01 kind: Event lastTimestamp: "2026-09-09T20:34:03Z" message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp 10.132.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:34:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:03Z" name: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2.18d3c153fc9296da namespace: kserve-ci-e2e-test resourceVersion: "28133" uid: 30f9e41f-6698-47f8-b35b-affb90427701 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3a010-predictor-7d55bcfc76 namespace: kserve-ci-e2e-test resourceVersion: "23474" uid: 8cdfc670-2578-46c5-9b91-b6a617f997af kind: Event lastTimestamp: "2026-09-09T20:24:45Z" message: 'Created pod: success-200-isvc-3a010-predictor-7d55bcfc76-5ggw2' metadata: creationTimestamp: "2026-09-09T20:24:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:45Z" name: success-200-isvc-3a010-predictor-7d55bcfc76.18d3c0d21c395d4f namespace: kserve-ci-e2e-test resourceVersion: "23481" uid: a2e2b802-6113-4f88-9b3a-aea28e52d12d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3a010-predictor namespace: kserve-ci-e2e-test resourceVersion: "23472" uid: a34611f9-9078-44f1-ab54-8210b3926373 kind: Event lastTimestamp: "2026-09-09T20:24:45Z" message: Scaled up replica set success-200-isvc-3a010-predictor-7d55bcfc76 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:24:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:45Z" name: success-200-isvc-3a010-predictor.18d3c0d21a80df86 namespace: kserve-ci-e2e-test resourceVersion: "23475" uid: 8c44e95e-3978-445b-ac75-e1af25e07be9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:25:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a010-predictor namespace: kserve-ci-e2e-test resourceVersion: "23473" uid: ce0e0fba-0535-43fe-888b-2f35c4313ff5 kind: Event lastTimestamp: "2026-09-09T20:25:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:25:15Z" name: success-200-isvc-3a010-predictor.18d3c0d599026c1a namespace: kserve-ci-e2e-test resourceVersion: "23830" uid: af3218a5-58d5-4d13-8dfe-56802c432084 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:25:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a010-predictor namespace: kserve-ci-e2e-test resourceVersion: "23473" uid: ce0e0fba-0535-43fe-888b-2f35c4313ff5 kind: Event lastTimestamp: "2026-09-09T20:25:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:25:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:25:15Z" name: success-200-isvc-3a010-predictor.18d3c0d599668b06 namespace: kserve-ci-e2e-test resourceVersion: "23831" uid: 895008e0-d37d-4594-9062-fbf79777e690 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:25:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a010-predictor namespace: kserve-ci-e2e-test resourceVersion: "23705" uid: ce0e0fba-0535-43fe-888b-2f35c4313ff5 kind: Event lastTimestamp: "2026-09-09T20:25:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:25:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:25:45Z" name: success-200-isvc-3a010-predictor.18d3c0dc96f2d597 namespace: kserve-ci-e2e-test resourceVersion: "24156" uid: 177f5253-f4ea-49b6-afed-4be6a0a4b552 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:25:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a010-predictor namespace: kserve-ci-e2e-test resourceVersion: "23705" uid: ce0e0fba-0535-43fe-888b-2f35c4313ff5 kind: Event lastTimestamp: "2026-09-09T20:25:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:25:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:25:45Z" name: success-200-isvc-3a010-predictor.18d3c0dc97592531 namespace: kserve-ci-e2e-test resourceVersion: "24159" uid: 08fbd822-24bb-42ce-be73-b18ab0c0be0e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:24:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3a010 namespace: kserve-ci-e2e-test resourceVersion: "23463" uid: d935250c-888e-4768-9d17-967cf306de80 kind: Event lastTimestamp: "2026-09-09T20:25:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:24:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:25:33Z" name: success-200-isvc-3a010.18d3c0d21a85f84a namespace: kserve-ci-e2e-test resourceVersion: "23984" uid: 3fb2f551-9b6b-4dd1-b6f7-b530805fe0bc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:25:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3a010 namespace: kserve-ci-e2e-test resourceVersion: "23981" uid: d935250c-888e-4768-9d17-967cf306de80 kind: Event lastTimestamp: "2026-09-09T20:25:33Z" message: InferenceService [success-200-isvc-3a010] is Ready metadata: creationTimestamp: "2026-09-09T20:25:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:25:33Z" name: success-200-isvc-3a010.18d3c0dd47279bd5 namespace: kserve-ci-e2e-test resourceVersion: "23982" uid: 510addb0-0ce1-462e-92f3-b55121c0b726 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:11:50.563020Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16456" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3b763-predictor-84bbbdf8d-mj65k to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:11:50Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c01da5b6e3c0 namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: ec55d054-7dbf-4855-a87f-77d709286fbd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:11:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-3b763-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:50Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c01db5c490c8 namespace: kserve-ci-e2e-test resourceVersion: "16523" uid: 3fae2ee6-eb65-4b9e-8e32-83cd57ad960d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:11:51Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:11:51Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c01deb48c35a namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: adc5fb4a-56a5-46d2-8499-8f065974b609 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:11:51Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1907" metadata: creationTimestamp: "2026-09-09T20:11:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:11:51Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c01deca9b80f namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: fbf595d3-6fc3-4966-9d5d-43eaf5ead5ed reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:12:07Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1907" in 15.895s (15.895s including waiting). Image size: 1356695112 bytes.' metadata: creationTimestamp: "2026-09-09T20:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:07Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c021a01a8dfc namespace: kserve-ci-e2e-test resourceVersion: "16745" uid: 2ddcc156-e0f9-4c40-8714-6cf3134d0df2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:12:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:07Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c021a385db94 namespace: kserve-ci-e2e-test resourceVersion: "16746" uid: 4aefc1c9-439c-4598-bfe7-5150deecebbe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:12:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:07Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c021a4029641 namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: cfb54868-5dfd-4e93-b7dd-61559e672e66 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:12:07Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-09T20:12:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:07Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c021a4130c9c namespace: kserve-ci-e2e-test resourceVersion: "16751" uid: d5e1ea30-9025-43e1-8788-0bdaf5abc347 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:12:10Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.616s (2.616s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-09T20:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:10Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c02240017c45 namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: e0e72a12-fc26-4543-953d-e20484fc360d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:12:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:10Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c02243b2e84f namespace: kserve-ci-e2e-test resourceVersion: "16777" uid: c716bb95-706b-46b3-8235-d1d9ddf9e479 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:12:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:12:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:12:10Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c0224447ee6d namespace: kserve-ci-e2e-test resourceVersion: "16779" uid: e7fdc6bf-ceee-4716-bd1e-bc1042eb4421 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-09T20:12:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:13:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:27Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c0227d0230a7 namespace: kserve-ci-e2e-test resourceVersion: "17631" uid: 5ca4316f-e339-4608-b235-911656a6f935 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:13:24Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:24Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c0338ecaf5d3 namespace: kserve-ci-e2e-test resourceVersion: "17536" uid: cc5daf92-da7a-4c2f-b0a1-b622a48778b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:13:24Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:24Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c0338ecbc533 namespace: kserve-ci-e2e-test resourceVersion: "17537" uid: 49b935d1-b774-478e-95c8-e409d564a8c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: bf756f90-33aa-462b-93c7-852055fbb991 kind: Event lastTimestamp: "2026-09-09T20:13:27Z" message: 'Readiness probe failed: Get "https://10.132.0.17:8643/healthz": dial tcp 10.132.0.17:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:13:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:27Z" name: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k.18d3c0342f2fd0f7 namespace: kserve-ci-e2e-test resourceVersion: "17630" uid: 42c8cf10-67a6-4184-bc72-bfd3c1977802 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3b763-predictor-84bbbdf8d namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: b3fb92f4-0659-4682-a944-8de59283b510 kind: Event lastTimestamp: "2026-09-09T20:11:50Z" message: 'Created pod: success-200-isvc-3b763-predictor-84bbbdf8d-mj65k' metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:11:50Z" name: success-200-isvc-3b763-predictor-84bbbdf8d.18d3c01da5482621 namespace: kserve-ci-e2e-test resourceVersion: "16457" uid: 6eb4e2d2-7a3b-4f7b-8a85-4eaf980a47f0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3b763-predictor namespace: kserve-ci-e2e-test resourceVersion: "16446" uid: 76e76bc8-8fce-4c8a-a827-6a20ca2e87c5 kind: Event lastTimestamp: "2026-09-09T20:11:50Z" message: Scaled up replica set success-200-isvc-3b763-predictor-84bbbdf8d from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:11:50Z" name: success-200-isvc-3b763-predictor.18d3c01da278a053 namespace: kserve-ci-e2e-test resourceVersion: "16450" uid: 7ebdcd15-e25f-409e-b37e-4bc4a017f89d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3b763-predictor namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: a18d91c8-dc9c-4050-8dae-4e019b0d003a kind: Event lastTimestamp: "2026-09-09T20:12:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:12:20Z" name: success-200-isvc-3b763-predictor.18d3c02120ff5fab namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: f22e57a0-52d8-49a7-bf4b-2fc5dbb5b4cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:12:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3b763-predictor namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: a18d91c8-dc9c-4050-8dae-4e019b0d003a kind: Event lastTimestamp: "2026-09-09T20:12:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:12:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:12:20Z" name: success-200-isvc-3b763-predictor.18d3c02121669d5b namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: c782d4c5-b9bf-47fa-9c72-77dfdfface9e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-09T20:12:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3b763-predictor namespace: kserve-ci-e2e-test resourceVersion: "16721" uid: a18d91c8-dc9c-4050-8dae-4e019b0d003a kind: Event lastTimestamp: "2026-09-09T20:13:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:20Z" name: success-200-isvc-3b763-predictor.18d3c0281ecdc3fe namespace: kserve-ci-e2e-test resourceVersion: "17432" uid: 89a92b04-1244-4fbb-9b14-3fa566605751 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-09T20:12:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3b763-predictor namespace: kserve-ci-e2e-test resourceVersion: "16721" uid: a18d91c8-dc9c-4050-8dae-4e019b0d003a kind: Event lastTimestamp: "2026-09-09T20:13:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:12:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:20Z" name: success-200-isvc-3b763-predictor.18d3c0281f30e308 namespace: kserve-ci-e2e-test resourceVersion: "17433" uid: 3a0ec0e6-4f4e-4918-96e0-8de8b8772021 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:11:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3b763 namespace: kserve-ci-e2e-test resourceVersion: "16439" uid: 1bc81560-b311-4f74-8ad0-2ff04ccc8c33 kind: Event lastTimestamp: "2026-09-09T20:12:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:11:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:12:57Z" name: success-200-isvc-3b763.18d3c01da2ae78ad namespace: kserve-ci-e2e-test resourceVersion: "17200" uid: c328437d-9aad-43e9-b781-d2c5dec82234 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:12:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3b763 namespace: kserve-ci-e2e-test resourceVersion: "17198" uid: 1bc81560-b311-4f74-8ad0-2ff04ccc8c33 kind: Event lastTimestamp: "2026-09-09T20:12:57Z" message: InferenceService [success-200-isvc-3b763] is Ready metadata: creationTimestamp: "2026-09-09T20:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:12:57Z" name: success-200-isvc-3b763.18d3c02d388d57a5 namespace: kserve-ci-e2e-test resourceVersion: "17199" uid: 097f9841-aa33-47bc-9dfc-2caa05185e49 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:23:49.735309Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3c295-predictor-c97b4b754-44xk4 namespace: kserve-ci-e2e-test resourceVersion: "22809" uid: 989b24c7-2c45-46e5-82aa-556c7ebcdb0e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3c295-predictor-c97b4b754-44xk4 to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:23:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:23:49Z" name: success-200-isvc-3c295-predictor-c97b4b754-44xk4.18d3c0c517b91652 namespace: kserve-ci-e2e-test resourceVersion: "22814" uid: 051e2d14-735d-435b-9d24-340d5026faec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3c295-predictor-c97b4b754-44xk4 namespace: kserve-ci-e2e-test resourceVersion: "22818" uid: 989b24c7-2c45-46e5-82aa-556c7ebcdb0e kind: Event lastTimestamp: "2026-09-09T20:23:50Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:23:50Z" name: success-200-isvc-3c295-predictor-c97b4b754-44xk4.18d3c0c5397f543d namespace: kserve-ci-e2e-test resourceVersion: "22854" uid: 9bf177a3-d77c-4646-846f-c32021b4df2d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c295-predictor-c97b4b754-44xk4 namespace: kserve-ci-e2e-test resourceVersion: "22811" uid: 989b24c7-2c45-46e5-82aa-556c7ebcdb0e kind: Event lastTimestamp: "2026-09-09T20:23:50Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:50Z" name: success-200-isvc-3c295-predictor-c97b4b754-44xk4.18d3c0c53ac723ed namespace: kserve-ci-e2e-test resourceVersion: "22858" uid: 052bfb74-8bb8-48a5-9e48-2ee1384acad5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c295-predictor-c97b4b754-44xk4 namespace: kserve-ci-e2e-test resourceVersion: "22811" uid: 989b24c7-2c45-46e5-82aa-556c7ebcdb0e kind: Event lastTimestamp: "2026-09-09T20:23:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:50Z" name: success-200-isvc-3c295-predictor-c97b4b754-44xk4.18d3c0c53e744169 namespace: kserve-ci-e2e-test resourceVersion: "22860" uid: 6a72a009-c0ea-4480-88c2-6c4d170445cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c295-predictor-c97b4b754-44xk4 namespace: kserve-ci-e2e-test resourceVersion: "22811" uid: 989b24c7-2c45-46e5-82aa-556c7ebcdb0e kind: Event lastTimestamp: "2026-09-09T20:23:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:50Z" name: success-200-isvc-3c295-predictor-c97b4b754-44xk4.18d3c0c53ee523c0 namespace: kserve-ci-e2e-test resourceVersion: "22861" uid: c5c8ae45-3fcb-44ce-9718-277f32ae9940 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c295-predictor-c97b4b754-44xk4 namespace: kserve-ci-e2e-test resourceVersion: "22811" uid: 989b24c7-2c45-46e5-82aa-556c7ebcdb0e kind: Event lastTimestamp: "2026-09-09T20:23:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:50Z" name: success-200-isvc-3c295-predictor-c97b4b754-44xk4.18d3c0c53eefe1de namespace: kserve-ci-e2e-test resourceVersion: "22862" uid: 14f4e946-d637-4a8d-9016-b63e50895436 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c295-predictor-c97b4b754-44xk4 namespace: kserve-ci-e2e-test resourceVersion: "22811" uid: 989b24c7-2c45-46e5-82aa-556c7ebcdb0e kind: Event lastTimestamp: "2026-09-09T20:23:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:50Z" name: success-200-isvc-3c295-predictor-c97b4b754-44xk4.18d3c0c541b80c5b namespace: kserve-ci-e2e-test resourceVersion: "22863" uid: 01c3b983-f0af-4606-95db-336730a6017d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c295-predictor-c97b4b754-44xk4 namespace: kserve-ci-e2e-test resourceVersion: "22811" uid: 989b24c7-2c45-46e5-82aa-556c7ebcdb0e kind: Event lastTimestamp: "2026-09-09T20:23:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:23:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:50Z" name: success-200-isvc-3c295-predictor-c97b4b754-44xk4.18d3c0c542467f2a namespace: kserve-ci-e2e-test resourceVersion: "22864" uid: 4827b0e1-3372-4b11-bd8a-c9dd9fcf460c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:23:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c295-predictor-c97b4b754-44xk4 namespace: kserve-ci-e2e-test resourceVersion: "22811" uid: 989b24c7-2c45-46e5-82aa-556c7ebcdb0e kind: Event lastTimestamp: "2026-09-09T20:24:28Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:23:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:28Z" name: success-200-isvc-3c295-predictor-c97b4b754-44xk4.18d3c0c5ab83a1c7 namespace: kserve-ci-e2e-test resourceVersion: "23220" uid: adc5f714-f2cd-44ee-add1-79d10258a132 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3c295-predictor-c97b4b754-44xk4 namespace: kserve-ci-e2e-test resourceVersion: "22811" uid: 989b24c7-2c45-46e5-82aa-556c7ebcdb0e kind: Event lastTimestamp: "2026-09-09T20:33:04Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:04Z" name: success-200-isvc-3c295-predictor-c97b4b754-44xk4.18d3c14640bd2771 namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: 1a211718-a141-4b87-bf83-e38df5213231 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3c295-predictor-c97b4b754-44xk4 namespace: kserve-ci-e2e-test resourceVersion: "22811" uid: 989b24c7-2c45-46e5-82aa-556c7ebcdb0e kind: Event lastTimestamp: "2026-09-09T20:33:04Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:04Z" name: success-200-isvc-3c295-predictor-c97b4b754-44xk4.18d3c14640bda7ad namespace: kserve-ci-e2e-test resourceVersion: "27442" uid: a99cd850-9bb8-47ae-bb02-d15949ad71e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3c295-predictor-c97b4b754 namespace: kserve-ci-e2e-test resourceVersion: "22803" uid: 2667358e-3e3d-4afa-a884-986e468f87e0 kind: Event lastTimestamp: "2026-09-09T20:23:49Z" message: 'Created pod: success-200-isvc-3c295-predictor-c97b4b754-44xk4' metadata: creationTimestamp: "2026-09-09T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:49Z" name: success-200-isvc-3c295-predictor-c97b4b754.18d3c0c5174cda5a namespace: kserve-ci-e2e-test resourceVersion: "22810" uid: aebc4dd7-17e9-4b09-b4ce-7a4dff2235cf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3c295-predictor namespace: kserve-ci-e2e-test resourceVersion: "22801" uid: e013621f-fdb4-4910-93da-6affadb4c3b0 kind: Event lastTimestamp: "2026-09-09T20:23:49Z" message: Scaled up replica set success-200-isvc-3c295-predictor-c97b4b754 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:49Z" name: success-200-isvc-3c295-predictor.18d3c0c515ae80e9 namespace: kserve-ci-e2e-test resourceVersion: "22804" uid: f5c07156-15d8-4d24-b793-4319f7e8319d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:24:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c295-predictor namespace: kserve-ci-e2e-test resourceVersion: "22802" uid: 9c47df29-5fde-4dc2-8a41-bbb23e8a77b6 kind: Event lastTimestamp: "2026-09-09T20:24:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:19Z" name: success-200-isvc-3c295-predictor.18d3c0c89487eee8 namespace: kserve-ci-e2e-test resourceVersion: "23145" uid: 5f201732-6af3-46cd-9aa0-d24a22eaf5eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:24:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c295-predictor namespace: kserve-ci-e2e-test resourceVersion: "22802" uid: 9c47df29-5fde-4dc2-8a41-bbb23e8a77b6 kind: Event lastTimestamp: "2026-09-09T20:24:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:24:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:19Z" name: success-200-isvc-3c295-predictor.18d3c0c894f181f5 namespace: kserve-ci-e2e-test resourceVersion: "23146" uid: 0f258899-8ed1-4740-938c-162501319285 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:24:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c295-predictor namespace: kserve-ci-e2e-test resourceVersion: "23010" uid: 9c47df29-5fde-4dc2-8a41-bbb23e8a77b6 kind: Event lastTimestamp: "2026-09-09T20:25:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:25:04Z" name: success-200-isvc-3c295-predictor.18d3c0cf92a6de42 namespace: kserve-ci-e2e-test resourceVersion: "23746" uid: 54f1b926-3a51-43e3-882f-c2b5592faca3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:24:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3c295-predictor namespace: kserve-ci-e2e-test resourceVersion: "23010" uid: 9c47df29-5fde-4dc2-8a41-bbb23e8a77b6 kind: Event lastTimestamp: "2026-09-09T20:25:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:24:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:25:04Z" name: success-200-isvc-3c295-predictor.18d3c0cf930be5b1 namespace: kserve-ci-e2e-test resourceVersion: "23747" uid: bd923b34-5dd6-47cf-82d2-46d5baff4b57 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:23:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3c295 namespace: kserve-ci-e2e-test resourceVersion: "22792" uid: d4f11ab4-2af9-4b30-be4c-a4fd376e5c32 kind: Event lastTimestamp: "2026-09-09T20:24:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:24:38Z" name: success-200-isvc-3c295.18d3c0c515b6102a namespace: kserve-ci-e2e-test resourceVersion: "23348" uid: 1b605623-e919-45f8-86b5-6d54e5003f45 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3c295 namespace: kserve-ci-e2e-test resourceVersion: "23343" uid: d4f11ab4-2af9-4b30-be4c-a4fd376e5c32 kind: Event lastTimestamp: "2026-09-09T20:24:38Z" message: InferenceService [success-200-isvc-3c295] is Ready metadata: creationTimestamp: "2026-09-09T20:24:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:24:38Z" name: success-200-isvc-3c295.18d3c0d066bfcd75 namespace: kserve-ci-e2e-test resourceVersion: "23344" uid: f2069952-78c8-4ff6-929f-02344eb0d1e2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:34:00.529040Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk namespace: kserve-ci-e2e-test resourceVersion: "28047" uid: e0879e79-177e-48a8-8992-e60a36b9898d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:34:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:34:00Z" name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk.18d3c1534dddcfd9 namespace: kserve-ci-e2e-test resourceVersion: "28052" uid: 2b844008-4d8d-40a4-b127-aa99b6ca7b96 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk namespace: kserve-ci-e2e-test resourceVersion: "28059" uid: e0879e79-177e-48a8-8992-e60a36b9898d kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:34:01Z" name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk.18d3c1536fcaac95 namespace: kserve-ci-e2e-test resourceVersion: "28094" uid: c8e74ba1-9b88-4223-9f37-bd0eaf03f8ac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk namespace: kserve-ci-e2e-test resourceVersion: "28050" uid: e0879e79-177e-48a8-8992-e60a36b9898d kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:01Z" name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk.18d3c153712ff115 namespace: kserve-ci-e2e-test resourceVersion: "28096" uid: f0f81e73-6903-443c-941c-d30ce2ab3be9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk namespace: kserve-ci-e2e-test resourceVersion: "28050" uid: e0879e79-177e-48a8-8992-e60a36b9898d kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:01Z" name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk.18d3c15373ac6562 namespace: kserve-ci-e2e-test resourceVersion: "28097" uid: a203b786-ccb2-4a13-91c8-aca11db31016 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk namespace: kserve-ci-e2e-test resourceVersion: "28050" uid: e0879e79-177e-48a8-8992-e60a36b9898d kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:01Z" name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk.18d3c153741de26e namespace: kserve-ci-e2e-test resourceVersion: "28098" uid: 6e872180-6b71-4050-8863-79af16c28a64 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk namespace: kserve-ci-e2e-test resourceVersion: "28050" uid: e0879e79-177e-48a8-8992-e60a36b9898d kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:01Z" name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk.18d3c1537428fa42 namespace: kserve-ci-e2e-test resourceVersion: "28099" uid: 59312e80-e128-4018-8315-f4bf32f250f0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk namespace: kserve-ci-e2e-test resourceVersion: "28050" uid: e0879e79-177e-48a8-8992-e60a36b9898d kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:01Z" name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk.18d3c15378e2afd7 namespace: kserve-ci-e2e-test resourceVersion: "28103" uid: 322cda50-4415-484d-a857-a4d3db7da0d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk namespace: kserve-ci-e2e-test resourceVersion: "28050" uid: e0879e79-177e-48a8-8992-e60a36b9898d kind: Event lastTimestamp: "2026-09-09T20:34:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:01Z" name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk.18d3c153795be1f3 namespace: kserve-ci-e2e-test resourceVersion: "28104" uid: 7df58546-bc01-4b73-83f9-330a35bcf5a6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-09T20:34:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk namespace: kserve-ci-e2e-test resourceVersion: "28050" uid: e0879e79-177e-48a8-8992-e60a36b9898d kind: Event lastTimestamp: "2026-09-09T20:34:47Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:47Z" name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk.18d3c153921ad563 namespace: kserve-ci-e2e-test resourceVersion: "28699" uid: 90ce8866-a96a-4d34-b2e4-c67d289376df reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:51:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk namespace: kserve-ci-e2e-test resourceVersion: "28050" uid: e0879e79-177e-48a8-8992-e60a36b9898d kind: Event lastTimestamp: "2026-09-09T20:51:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:51:29Z" name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk.18d3c24799ce3f60 namespace: kserve-ci-e2e-test resourceVersion: "35824" uid: 329262c6-df00-4ca7-8693-9b64e0d3a478 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:51:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk namespace: kserve-ci-e2e-test resourceVersion: "28050" uid: e0879e79-177e-48a8-8992-e60a36b9898d kind: Event lastTimestamp: "2026-09-09T20:51:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:51:29Z" name: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk.18d3c24799cebace namespace: kserve-ci-e2e-test resourceVersion: "35830" uid: 77ccb8eb-2e13-4bc7-b8f3-23ad5aa562ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-54686-predictor-5b5bdc6d9b namespace: kserve-ci-e2e-test resourceVersion: "28040" uid: f10add42-5ed9-40c5-9b93-08c40a0f2d54 kind: Event lastTimestamp: "2026-09-09T20:34:00Z" message: 'Created pod: success-200-isvc-54686-predictor-5b5bdc6d9b-7tchk' metadata: creationTimestamp: "2026-09-09T20:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:00Z" name: success-200-isvc-54686-predictor-5b5bdc6d9b.18d3c1534d45f9d2 namespace: kserve-ci-e2e-test resourceVersion: "28049" uid: 4386fb67-0eda-4edc-9042-d8e279ceed3a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-54686-predictor namespace: kserve-ci-e2e-test resourceVersion: "28038" uid: 42a35f93-b4e0-43a0-892f-b2b51d79a0d4 kind: Event lastTimestamp: "2026-09-09T20:34:00Z" message: Scaled up replica set success-200-isvc-54686-predictor-5b5bdc6d9b from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:00Z" name: success-200-isvc-54686-predictor.18d3c1534ad80e19 namespace: kserve-ci-e2e-test resourceVersion: "28041" uid: 818bb46f-7bad-4857-b2e9-7e456e5829f4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-54686-predictor namespace: kserve-ci-e2e-test resourceVersion: "28039" uid: be699c53-e333-42f5-a982-d908593abb39 kind: Event lastTimestamp: "2026-09-09T20:34:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:30Z" name: success-200-isvc-54686-predictor.18d3c156c9650e8d namespace: kserve-ci-e2e-test resourceVersion: "28552" uid: a1a79da7-f119-4ddb-b4b5-d87ae4fe1ffa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-54686-predictor namespace: kserve-ci-e2e-test resourceVersion: "28039" uid: be699c53-e333-42f5-a982-d908593abb39 kind: Event lastTimestamp: "2026-09-09T20:34:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:30Z" name: success-200-isvc-54686-predictor.18d3c156c9eea36e namespace: kserve-ci-e2e-test resourceVersion: "28553" uid: 8502ea04-7aad-475e-99d2-c121dcda0441 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:34:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-54686-predictor namespace: kserve-ci-e2e-test resourceVersion: "28390" uid: be699c53-e333-42f5-a982-d908593abb39 kind: Event lastTimestamp: "2026-09-09T20:35:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:15Z" name: success-200-isvc-54686-predictor.18d3c15dc75d009a namespace: kserve-ci-e2e-test resourceVersion: "28976" uid: 9103013e-8c32-4dac-8d73-e35f982b0f35 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:34:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-54686-predictor namespace: kserve-ci-e2e-test resourceVersion: "28390" uid: be699c53-e333-42f5-a982-d908593abb39 kind: Event lastTimestamp: "2026-09-09T20:35:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:34:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:15Z" name: success-200-isvc-54686-predictor.18d3c15dc7bdfde8 namespace: kserve-ci-e2e-test resourceVersion: "28977" uid: 48910808-7d2c-44b6-9308-97f938ace1d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:34:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-54686 namespace: kserve-ci-e2e-test resourceVersion: "28028" uid: b8ac4fe0-c07a-460b-9025-6302c6db492c kind: Event lastTimestamp: "2026-09-09T20:34:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:34:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:34:57Z" name: success-200-isvc-54686.18d3c1534ae08079 namespace: kserve-ci-e2e-test resourceVersion: "28784" uid: ec0995f4-99ec-428a-962f-caeef1c52612 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-54686 namespace: kserve-ci-e2e-test resourceVersion: "28782" uid: b8ac4fe0-c07a-460b-9025-6302c6db492c kind: Event lastTimestamp: "2026-09-09T20:34:57Z" message: InferenceService [success-200-isvc-54686] is Ready metadata: creationTimestamp: "2026-09-09T20:34:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:34:57Z" name: success-200-isvc-54686.18d3c160a1496d34 namespace: kserve-ci-e2e-test resourceVersion: "28783" uid: 7dfaa9c2-73fb-424f-80f0-cefdc675e009 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:33:04.621346Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb namespace: kserve-ci-e2e-test resourceVersion: "27463" uid: ddeb1746-bbcc-463f-a3f3-dd7d089e0cb4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:33:04Z" name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb.18d3c146498235a6 namespace: kserve-ci-e2e-test resourceVersion: "27473" uid: 95ab5213-f3b4-4430-953d-36bee2472b66 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb namespace: kserve-ci-e2e-test resourceVersion: "27490" uid: ddeb1746-bbcc-463f-a3f3-dd7d089e0cb4 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:33:05Z" name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb.18d3c1466b8bc2cf namespace: kserve-ci-e2e-test resourceVersion: "27521" uid: 6afd815b-26b1-465a-a651-f050335ba89d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: ddeb1746-bbcc-463f-a3f3-dd7d089e0cb4 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:05Z" name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb.18d3c1466cefe4e6 namespace: kserve-ci-e2e-test resourceVersion: "27523" uid: 8d96ec88-3bb9-4def-a390-4e85b5c0dc31 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: ddeb1746-bbcc-463f-a3f3-dd7d089e0cb4 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:05Z" name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb.18d3c1466f7b22c6 namespace: kserve-ci-e2e-test resourceVersion: "27524" uid: 3961d8a6-d2a1-466c-a913-41b5ae698d33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: ddeb1746-bbcc-463f-a3f3-dd7d089e0cb4 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:05Z" name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb.18d3c1466fe968e4 namespace: kserve-ci-e2e-test resourceVersion: "27525" uid: 25056966-cb78-4641-ae66-8422457aa9b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: ddeb1746-bbcc-463f-a3f3-dd7d089e0cb4 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:05Z" name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb.18d3c1466ff2a3b9 namespace: kserve-ci-e2e-test resourceVersion: "27526" uid: da2b5af5-44f1-43eb-ae84-75e26078fa2f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: ddeb1746-bbcc-463f-a3f3-dd7d089e0cb4 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:05Z" name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb.18d3c146729e9b9f namespace: kserve-ci-e2e-test resourceVersion: "27530" uid: 86ae3765-f1d9-4e1f-a9c5-93f6f89db204 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: ddeb1746-bbcc-463f-a3f3-dd7d089e0cb4 kind: Event lastTimestamp: "2026-09-09T20:33:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:33:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:05Z" name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb.18d3c146732d289f namespace: kserve-ci-e2e-test resourceVersion: "27531" uid: 3802c03d-d348-417e-a6dc-4b6a5b8fa025 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:33:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: ddeb1746-bbcc-463f-a3f3-dd7d089e0cb4 kind: Event lastTimestamp: "2026-09-09T20:33:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:33:42Z" name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb.18d3c146b57ceed6 namespace: kserve-ci-e2e-test resourceVersion: "27834" uid: 009a91d0-0cbe-4c69-b2e9-b4b03e20aa2f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: ddeb1746-bbcc-463f-a3f3-dd7d089e0cb4 kind: Event lastTimestamp: "2026-09-09T20:34:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:14Z" name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb.18d3c156a09c9237 namespace: kserve-ci-e2e-test resourceVersion: "28307" uid: 019cc47f-2a6d-4766-9cc1-9e60d3472c94 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: ddeb1746-bbcc-463f-a3f3-dd7d089e0cb4 kind: Event lastTimestamp: "2026-09-09T20:34:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:14Z" name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb.18d3c156a09d1809 namespace: kserve-ci-e2e-test resourceVersion: "28309" uid: cb0ca841-da75-496b-b571-35da296c9bf1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: ddeb1746-bbcc-463f-a3f3-dd7d089e0cb4 kind: Event lastTimestamp: "2026-09-09T20:34:17Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:34:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:17Z" name: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb.18d3c1573da14eaf namespace: kserve-ci-e2e-test resourceVersion: "28439" uid: 460f8736-e903-4c61-8c99-cc18ac304509 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-97f7b-predictor-6f5d8d9788 namespace: kserve-ci-e2e-test resourceVersion: "27449" uid: deede564-aa18-4aeb-8501-0636d1c244d9 kind: Event lastTimestamp: "2026-09-09T20:33:04Z" message: 'Created pod: success-200-isvc-97f7b-predictor-6f5d8d9788-zbckb' metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:33:04Z" name: success-200-isvc-97f7b-predictor-6f5d8d9788.18d3c146492ea895 namespace: kserve-ci-e2e-test resourceVersion: "27468" uid: 806ba59e-b8c6-42c6-9588-7a55712cd5f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-97f7b-predictor namespace: kserve-ci-e2e-test resourceVersion: "27446" uid: f28a7fe3-7f40-4682-a154-89d249d05447 kind: Event lastTimestamp: "2026-09-09T20:33:04Z" message: Scaled up replica set success-200-isvc-97f7b-predictor-6f5d8d9788 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:33:04Z" name: success-200-isvc-97f7b-predictor.18d3c146434bfda3 namespace: kserve-ci-e2e-test resourceVersion: "27451" uid: 7dca454a-5fdf-4246-9d30-d49c572ba5e5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:33:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-97f7b-predictor namespace: kserve-ci-e2e-test resourceVersion: "27448" uid: 2f5244e7-407d-4676-bce8-c9d50dfe418e kind: Event lastTimestamp: "2026-09-09T20:33:34Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:33:34Z" name: success-200-isvc-97f7b-predictor.18d3c149c159c1b1 namespace: kserve-ci-e2e-test resourceVersion: "27772" uid: 15b54684-1117-4561-bfdf-75d35c2d81df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:33:19Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-97f7b-predictor namespace: kserve-ci-e2e-test resourceVersion: "27448" uid: 2f5244e7-407d-4676-bce8-c9d50dfe418e kind: Event lastTimestamp: "2026-09-09T20:33:34Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:33:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:33:34Z" name: success-200-isvc-97f7b-predictor.18d3c149c1b99451 namespace: kserve-ci-e2e-test resourceVersion: "27773" uid: ff6eb611-c988-4bd3-9857-dbcc68c51333 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:33:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-97f7b-predictor namespace: kserve-ci-e2e-test resourceVersion: "27660" uid: 2f5244e7-407d-4676-bce8-c9d50dfe418e kind: Event lastTimestamp: "2026-09-09T20:34:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:33:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:04Z" name: success-200-isvc-97f7b-predictor.18d3c150bf5bcff8 namespace: kserve-ci-e2e-test resourceVersion: "28155" uid: f8fecc3a-8575-48cf-9721-3b93bfff40e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:33:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-97f7b-predictor namespace: kserve-ci-e2e-test resourceVersion: "27660" uid: 2f5244e7-407d-4676-bce8-c9d50dfe418e kind: Event lastTimestamp: "2026-09-09T20:34:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:33:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:04Z" name: success-200-isvc-97f7b-predictor.18d3c150bfbde74f namespace: kserve-ci-e2e-test resourceVersion: "28157" uid: c7cb7857-4fde-490b-b99c-8d9f665a7caa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-97f7b namespace: kserve-ci-e2e-test resourceVersion: "27423" uid: e16febab-cb8c-4ff3-aac5-59d8fe40c55b kind: Event lastTimestamp: "2026-09-09T20:33:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:33:52Z" name: success-200-isvc-97f7b.18d3c14642f004d6 namespace: kserve-ci-e2e-test resourceVersion: "27929" uid: b3b3ef10-b02c-49a0-b328-f4c96f72e108 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-97f7b namespace: kserve-ci-e2e-test resourceVersion: "27423" uid: e16febab-cb8c-4ff3-aac5-59d8fe40c55b kind: Event lastTimestamp: "2026-09-09T20:33:04Z" message: 'Failed to update status for InferenceService "success-200-isvc-97f7b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-97f7b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:33:04Z" name: success-200-isvc-97f7b.18d3c1464361b04c namespace: kserve-ci-e2e-test resourceVersion: "27454" uid: 55d22d61-5937-406f-a16b-dc4db1369f2f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-97f7b namespace: kserve-ci-e2e-test resourceVersion: "27423" uid: e16febab-cb8c-4ff3-aac5-59d8fe40c55b kind: Event lastTimestamp: "2026-09-09T20:33:04Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-97f7b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-09T20:33:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:33:04Z" name: success-200-isvc-97f7b.18d3c1464361fade namespace: kserve-ci-e2e-test resourceVersion: "27455" uid: 95429cda-eda6-4d34-b831-217cdbee23f3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:33:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-97f7b namespace: kserve-ci-e2e-test resourceVersion: "27927" uid: e16febab-cb8c-4ff3-aac5-59d8fe40c55b kind: Event lastTimestamp: "2026-09-09T20:33:52Z" message: InferenceService [success-200-isvc-97f7b] is Ready metadata: creationTimestamp: "2026-09-09T20:33:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:33:52Z" name: success-200-isvc-97f7b.18d3c15170c3acbb namespace: kserve-ci-e2e-test resourceVersion: "27928" uid: 5b84fb19-2171-422d-8003-ea11dbbd2d55 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:34:14.945586Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj namespace: kserve-ci-e2e-test resourceVersion: "28341" uid: 06caf308-6682-41dd-a365-1c0a17cc0e6e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:34:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:34:14Z" name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj.18d3c156a928dd96 namespace: kserve-ci-e2e-test resourceVersion: "28345" uid: c66445ff-e888-4513-ab14-f3f0c5453a35 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj namespace: kserve-ci-e2e-test resourceVersion: "28350" uid: 06caf308-6682-41dd-a365-1c0a17cc0e6e kind: Event lastTimestamp: "2026-09-09T20:34:15Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:34:15Z" name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj.18d3c156cae554e3 namespace: kserve-ci-e2e-test resourceVersion: "28392" uid: 698e290c-49d2-489a-bd33-7ddce32b2f74 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj namespace: kserve-ci-e2e-test resourceVersion: "28343" uid: 06caf308-6682-41dd-a365-1c0a17cc0e6e kind: Event lastTimestamp: "2026-09-09T20:34:15Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:15Z" name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj.18d3c156cc333bb5 namespace: kserve-ci-e2e-test resourceVersion: "28394" uid: 539e5193-ccd2-4c99-9788-e67825be9252 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj namespace: kserve-ci-e2e-test resourceVersion: "28343" uid: 06caf308-6682-41dd-a365-1c0a17cc0e6e kind: Event lastTimestamp: "2026-09-09T20:34:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:15Z" name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj.18d3c156cf76a4bf namespace: kserve-ci-e2e-test resourceVersion: "28395" uid: 76b8f906-7a40-4592-bec7-36a89bcc6ce3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj namespace: kserve-ci-e2e-test resourceVersion: "28343" uid: 06caf308-6682-41dd-a365-1c0a17cc0e6e kind: Event lastTimestamp: "2026-09-09T20:34:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:15Z" name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj.18d3c156cfe66834 namespace: kserve-ci-e2e-test resourceVersion: "28398" uid: 36fed99d-15ad-4921-b584-63cf818f91e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj namespace: kserve-ci-e2e-test resourceVersion: "28343" uid: 06caf308-6682-41dd-a365-1c0a17cc0e6e kind: Event lastTimestamp: "2026-09-09T20:34:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:15Z" name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj.18d3c156cfefaf81 namespace: kserve-ci-e2e-test resourceVersion: "28400" uid: 377ee72d-69fa-4cdf-bfb7-81fbd5017207 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj namespace: kserve-ci-e2e-test resourceVersion: "28343" uid: 06caf308-6682-41dd-a365-1c0a17cc0e6e kind: Event lastTimestamp: "2026-09-09T20:34:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:15Z" name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj.18d3c156d2e8f4af namespace: kserve-ci-e2e-test resourceVersion: "28402" uid: 59390a27-0e1f-43e4-86bb-4b79bc43a924 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj namespace: kserve-ci-e2e-test resourceVersion: "28343" uid: 06caf308-6682-41dd-a365-1c0a17cc0e6e kind: Event lastTimestamp: "2026-09-09T20:34:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:34:15Z" name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj.18d3c156d376bd62 namespace: kserve-ci-e2e-test resourceVersion: "28403" uid: 871938d9-7d54-4594-bf43-2c28924ef133 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-09T20:34:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj namespace: kserve-ci-e2e-test resourceVersion: "28343" uid: 06caf308-6682-41dd-a365-1c0a17cc0e6e kind: Event lastTimestamp: "2026-09-09T20:35:02Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:34:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:35:02Z" name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj.18d3c156d90957eb namespace: kserve-ci-e2e-test resourceVersion: "28831" uid: b7738552-fb57-4a08-8dc0-9b786544f974 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj namespace: kserve-ci-e2e-test resourceVersion: "28343" uid: 06caf308-6682-41dd-a365-1c0a17cc0e6e kind: Event lastTimestamp: "2026-09-09T20:43:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:43:39Z" name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj.18d3c1da2601ff76 namespace: kserve-ci-e2e-test resourceVersion: "32552" uid: 6a66203a-491c-4f03-8b76-e9b2e2b8d6c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:43:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj namespace: kserve-ci-e2e-test resourceVersion: "28343" uid: 06caf308-6682-41dd-a365-1c0a17cc0e6e kind: Event lastTimestamp: "2026-09-09T20:43:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:43:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:43:39Z" name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj.18d3c1da26028d86 namespace: kserve-ci-e2e-test resourceVersion: "32553" uid: 19663832-ec42-443e-98c6-8144bce7704d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:43:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj namespace: kserve-ci-e2e-test resourceVersion: "28343" uid: 06caf308-6682-41dd-a365-1c0a17cc0e6e kind: Event lastTimestamp: "2026-09-09T20:43:42Z" message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": dial tcp 10.132.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:43:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:43:42Z" name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj.18d3c1daddddafca namespace: kserve-ci-e2e-test resourceVersion: "32593" uid: cb4f6212-6ddf-4b0f-8ece-fdbf73c812e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:43:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj namespace: kserve-ci-e2e-test resourceVersion: "28343" uid: 06caf308-6682-41dd-a365-1c0a17cc0e6e kind: Event lastTimestamp: "2026-09-09T20:43:43Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: i/o timeout' metadata: creationTimestamp: "2026-09-09T20:43:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:43:43Z" name: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj.18d3c1db19cd7a78 namespace: kserve-ci-e2e-test resourceVersion: "32608" uid: a37c2ecc-0144-4471-8c65-32fe9a400299 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b9ccb-predictor-869d5b846d namespace: kserve-ci-e2e-test resourceVersion: "28335" uid: 2d00f635-1c68-497b-a0c7-da456b8ecfc8 kind: Event lastTimestamp: "2026-09-09T20:34:14Z" message: 'Created pod: success-200-isvc-b9ccb-predictor-869d5b846d-8ssrj' metadata: creationTimestamp: "2026-09-09T20:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:14Z" name: success-200-isvc-b9ccb-predictor-869d5b846d.18d3c156a8bcb473 namespace: kserve-ci-e2e-test resourceVersion: "28342" uid: fc5eeef6-6f42-4a42-8ddb-cd4689a24b45 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:34:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b9ccb-predictor namespace: kserve-ci-e2e-test resourceVersion: "28333" uid: 5a7a0343-9b5f-43a7-9127-afcaf8a7c9f6 kind: Event lastTimestamp: "2026-09-09T20:34:14Z" message: Scaled up replica set success-200-isvc-b9ccb-predictor-869d5b846d from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:14Z" name: success-200-isvc-b9ccb-predictor.18d3c156a6edc6ac namespace: kserve-ci-e2e-test resourceVersion: "28336" uid: a741d200-ee5f-4e7c-895f-bb09df2fd8f1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:34:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9ccb-predictor namespace: kserve-ci-e2e-test resourceVersion: "28334" uid: 3210f985-b399-4436-af62-a006d2805e69 kind: Event lastTimestamp: "2026-09-09T20:34:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:44Z" name: success-200-isvc-b9ccb-predictor.18d3c15a2576e501 namespace: kserve-ci-e2e-test resourceVersion: "28669" uid: 7503afb1-a59b-4fbf-be24-688bd07d2ae7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:34:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9ccb-predictor namespace: kserve-ci-e2e-test resourceVersion: "28334" uid: 3210f985-b399-4436-af62-a006d2805e69 kind: Event lastTimestamp: "2026-09-09T20:34:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:34:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:34:44Z" name: success-200-isvc-b9ccb-predictor.18d3c15a25c0803a namespace: kserve-ci-e2e-test resourceVersion: "28670" uid: a0617f3e-f7d9-4bfd-a873-af96bf0ed7d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:34:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9ccb-predictor namespace: kserve-ci-e2e-test resourceVersion: "28543" uid: 3210f985-b399-4436-af62-a006d2805e69 kind: Event lastTimestamp: "2026-09-09T20:35:29Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:34:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:29Z" name: success-200-isvc-b9ccb-predictor.18d3c1612368acb5 namespace: kserve-ci-e2e-test resourceVersion: "29119" uid: 539d81d2-2215-400a-b12b-9cd2898e967e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:34:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b9ccb-predictor namespace: kserve-ci-e2e-test resourceVersion: "28543" uid: 3210f985-b399-4436-af62-a006d2805e69 kind: Event lastTimestamp: "2026-09-09T20:35:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:34:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:29Z" name: success-200-isvc-b9ccb-predictor.18d3c16123d18211 namespace: kserve-ci-e2e-test resourceVersion: "29120" uid: 1d78807e-7143-4c60-8318-1ced51a40add reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:34:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9ccb namespace: kserve-ci-e2e-test resourceVersion: "28323" uid: 91c85501-ed87-49c6-8f85-14aa8a9da2bb kind: Event lastTimestamp: "2026-09-09T20:35:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:34:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:35:12Z" name: success-200-isvc-b9ccb.18d3c156a706dc46 namespace: kserve-ci-e2e-test resourceVersion: "28956" uid: dedd4a2d-9122-4965-b88c-8e532ccb649a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b9ccb namespace: kserve-ci-e2e-test resourceVersion: "28954" uid: 91c85501-ed87-49c6-8f85-14aa8a9da2bb kind: Event lastTimestamp: "2026-09-09T20:35:12Z" message: InferenceService [success-200-isvc-b9ccb] is Ready metadata: creationTimestamp: "2026-09-09T20:35:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:35:12Z" name: success-200-isvc-b9ccb.18d3c16424a5e247 namespace: kserve-ci-e2e-test resourceVersion: "28955" uid: 8035b92a-0a2d-445e-87a0-8d8ee1afd8fd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:23:25.361782Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr namespace: kserve-ci-e2e-test resourceVersion: "22415" uid: 58895f95-0127-4cde-bf11-7e427cbcd906 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bf2e8-predictor-c577cf68-h9ggr to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:23:25Z" name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr.18d3c0bf6af2a022 namespace: kserve-ci-e2e-test resourceVersion: "22420" uid: 420fac31-479c-4db0-b6ca-820de119f6f6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr namespace: kserve-ci-e2e-test resourceVersion: "22426" uid: 58895f95-0127-4cde-bf11-7e427cbcd906 kind: Event lastTimestamp: "2026-09-09T20:23:25Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:23:25Z" name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr.18d3c0bf8cc3782e namespace: kserve-ci-e2e-test resourceVersion: "22463" uid: 2078c7f7-7315-47c5-b41b-05de9a37e38d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 58895f95-0127-4cde-bf11-7e427cbcd906 kind: Event lastTimestamp: "2026-09-09T20:23:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:25Z" name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr.18d3c0bf8e1e2690 namespace: kserve-ci-e2e-test resourceVersion: "22465" uid: 1b7d0fa6-345d-4f72-9a56-c4e552796632 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 58895f95-0127-4cde-bf11-7e427cbcd906 kind: Event lastTimestamp: "2026-09-09T20:23:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:25Z" name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr.18d3c0bf90cf7dea namespace: kserve-ci-e2e-test resourceVersion: "22467" uid: a4b5e200-420b-42bb-b968-d5f9e098108a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 58895f95-0127-4cde-bf11-7e427cbcd906 kind: Event lastTimestamp: "2026-09-09T20:23:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:26Z" name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr.18d3c0bf913b26b9 namespace: kserve-ci-e2e-test resourceVersion: "22468" uid: 5b7820de-637e-4de9-a457-1cf9103f7444 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 58895f95-0127-4cde-bf11-7e427cbcd906 kind: Event lastTimestamp: "2026-09-09T20:23:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:26Z" name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr.18d3c0bf91445721 namespace: kserve-ci-e2e-test resourceVersion: "22469" uid: c9dc661d-00db-40d9-acbd-6d6a9aebd824 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 58895f95-0127-4cde-bf11-7e427cbcd906 kind: Event lastTimestamp: "2026-09-09T20:23:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:26Z" name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr.18d3c0bf9422781b namespace: kserve-ci-e2e-test resourceVersion: "22472" uid: 9e352277-1064-4108-8811-a7d190223890 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 58895f95-0127-4cde-bf11-7e427cbcd906 kind: Event lastTimestamp: "2026-09-09T20:23:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:26Z" name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr.18d3c0bf949c0a82 namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: 331c9617-0004-4812-8ec0-ef837896c60d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-09T20:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 58895f95-0127-4cde-bf11-7e427cbcd906 kind: Event lastTimestamp: "2026-09-09T20:24:12Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:12Z" name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr.18d3c0bf963d06b4 namespace: kserve-ci-e2e-test resourceVersion: "23073" uid: b8913698-fcb1-4116-bc12-a8d39cc1b8ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 58895f95-0127-4cde-bf11-7e427cbcd906 kind: Event lastTimestamp: "2026-09-09T20:24:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:24:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:45Z" name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr.18d3c0d214f893da namespace: kserve-ci-e2e-test resourceVersion: "23447" uid: ed8b1eb0-e8b1-4944-a962-10504566364e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 58895f95-0127-4cde-bf11-7e427cbcd906 kind: Event lastTimestamp: "2026-09-09T20:24:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:24:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:45Z" name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr.18d3c0d214f9383c namespace: kserve-ci-e2e-test resourceVersion: "23448" uid: 4623f273-216d-4d3c-87f8-32a096e147b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: 58895f95-0127-4cde-bf11-7e427cbcd906 kind: Event lastTimestamp: "2026-09-09T20:24:47Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:24:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:24:47Z" name: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr.18d3c0d272ba4d16 namespace: kserve-ci-e2e-test resourceVersion: "23553" uid: 30fba2ef-0600-4c7a-a98f-d6ff9b6fd272 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bf2e8-predictor-c577cf68 namespace: kserve-ci-e2e-test resourceVersion: "22409" uid: 885ca5e2-0445-40e4-be67-802471cb4ccf kind: Event lastTimestamp: "2026-09-09T20:23:25Z" message: 'Created pod: success-200-isvc-bf2e8-predictor-c577cf68-h9ggr' metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:25Z" name: success-200-isvc-bf2e8-predictor-c577cf68.18d3c0bf6a8d7577 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 8a261f84-d69a-48b1-bc1b-b31d1a0875ee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bf2e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22407" uid: 73963d35-9bbc-46b1-91ea-6299b912d8f9 kind: Event lastTimestamp: "2026-09-09T20:23:25Z" message: Scaled up replica set success-200-isvc-bf2e8-predictor-c577cf68 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:25Z" name: success-200-isvc-bf2e8-predictor.18d3c0bf68ecceb7 namespace: kserve-ci-e2e-test resourceVersion: "22410" uid: 29c7a158-cdaf-4256-953f-e2b4dee559c5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:23:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bf2e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22408" uid: 5878d9fe-50ba-4a4a-851d-bb964b050080 kind: Event lastTimestamp: "2026-09-09T20:23:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:55Z" name: success-200-isvc-bf2e8-predictor.18d3c0c2e7585442 namespace: kserve-ci-e2e-test resourceVersion: "22933" uid: 5faad98f-7acd-4bbb-aa94-0a987950cc27 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:23:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bf2e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22408" uid: 5878d9fe-50ba-4a4a-851d-bb964b050080 kind: Event lastTimestamp: "2026-09-09T20:23:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:23:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:55Z" name: success-200-isvc-bf2e8-predictor.18d3c0c2e7bb9e0b namespace: kserve-ci-e2e-test resourceVersion: "22934" uid: cee4a44f-35b4-4e81-afc5-8e764541895c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:24:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bf2e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22653" uid: 5878d9fe-50ba-4a4a-851d-bb964b050080 kind: Event lastTimestamp: "2026-09-09T20:24:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:40Z" name: success-200-isvc-bf2e8-predictor.18d3c0c9e51f6ce2 namespace: kserve-ci-e2e-test resourceVersion: "23363" uid: 278603b3-e3f9-42cc-aee3-99355bb4d3f3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:24:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bf2e8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22653" uid: 5878d9fe-50ba-4a4a-851d-bb964b050080 kind: Event lastTimestamp: "2026-09-09T20:24:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:24:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:24:40Z" name: success-200-isvc-bf2e8-predictor.18d3c0c9e56691b4 namespace: kserve-ci-e2e-test resourceVersion: "23364" uid: 2449d260-8904-4cda-9389-e67263bd7421 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:23:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bf2e8 namespace: kserve-ci-e2e-test resourceVersion: "22397" uid: a9d1c7dc-875f-455a-8196-935faf61cc5b kind: Event lastTimestamp: "2026-09-09T20:24:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:23:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:24:22Z" name: success-200-isvc-bf2e8.18d3c0bf68f50cec namespace: kserve-ci-e2e-test resourceVersion: "23175" uid: 6b105f2e-26e2-4b9a-9ba6-ff1a1a64d1a7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:24:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bf2e8 namespace: kserve-ci-e2e-test resourceVersion: "23173" uid: a9d1c7dc-875f-455a-8196-935faf61cc5b kind: Event lastTimestamp: "2026-09-09T20:24:22Z" message: InferenceService [success-200-isvc-bf2e8] is Ready metadata: creationTimestamp: "2026-09-09T20:24:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:24:22Z" name: success-200-isvc-bf2e8.18d3c0cca63afc88 namespace: kserve-ci-e2e-test resourceVersion: "23174" uid: 4390ec31-6d1e-416b-a3fd-88d229d75ac8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:13:24.636713Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d2727-predictor-775569db54-5zrm6 namespace: kserve-ci-e2e-test resourceVersion: "17513" uid: 73b3ca7a-0aeb-4dd4-bc64-f66f7c8abfe9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d2727-predictor-775569db54-5zrm6 to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:13:24Z" name: success-200-isvc-d2727-predictor-775569db54-5zrm6.18d3c0338cf1942e namespace: kserve-ci-e2e-test resourceVersion: "17525" uid: 5136b11e-eb49-4562-9f4f-7b71a90cf32f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d2727-predictor-775569db54-5zrm6 namespace: kserve-ci-e2e-test resourceVersion: "17535" uid: 73b3ca7a-0aeb-4dd4-bc64-f66f7c8abfe9 kind: Event lastTimestamp: "2026-09-09T20:13:25Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:13:25Z" name: success-200-isvc-d2727-predictor-775569db54-5zrm6.18d3c033aebda927 namespace: kserve-ci-e2e-test resourceVersion: "17588" uid: a4d9d9a2-1d48-41c9-b0dd-4dc9a75993d3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2727-predictor-775569db54-5zrm6 namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: 73b3ca7a-0aeb-4dd4-bc64-f66f7c8abfe9 kind: Event lastTimestamp: "2026-09-09T20:13:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:25Z" name: success-200-isvc-d2727-predictor-775569db54-5zrm6.18d3c033afee6a02 namespace: kserve-ci-e2e-test resourceVersion: "17591" uid: 3bb28bee-a6b7-4ee8-a977-7e8973ae3979 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2727-predictor-775569db54-5zrm6 namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: 73b3ca7a-0aeb-4dd4-bc64-f66f7c8abfe9 kind: Event lastTimestamp: "2026-09-09T20:13:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:25Z" name: success-200-isvc-d2727-predictor-775569db54-5zrm6.18d3c033b26dadcc namespace: kserve-ci-e2e-test resourceVersion: "17592" uid: b376bcc4-98e4-4642-974f-fb47dabbb2f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2727-predictor-775569db54-5zrm6 namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: 73b3ca7a-0aeb-4dd4-bc64-f66f7c8abfe9 kind: Event lastTimestamp: "2026-09-09T20:13:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:25Z" name: success-200-isvc-d2727-predictor-775569db54-5zrm6.18d3c033b2dc0710 namespace: kserve-ci-e2e-test resourceVersion: "17593" uid: 92e27dcf-762b-470c-8af2-a4f53df58062 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2727-predictor-775569db54-5zrm6 namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: 73b3ca7a-0aeb-4dd4-bc64-f66f7c8abfe9 kind: Event lastTimestamp: "2026-09-09T20:13:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:25Z" name: success-200-isvc-d2727-predictor-775569db54-5zrm6.18d3c033b2e54cdb namespace: kserve-ci-e2e-test resourceVersion: "17594" uid: 6ce412d0-7a77-4f13-aa51-1a770f18c1e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2727-predictor-775569db54-5zrm6 namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: 73b3ca7a-0aeb-4dd4-bc64-f66f7c8abfe9 kind: Event lastTimestamp: "2026-09-09T20:13:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:25Z" name: success-200-isvc-d2727-predictor-775569db54-5zrm6.18d3c033b5694dce namespace: kserve-ci-e2e-test resourceVersion: "17596" uid: 1e9dd925-15b2-4ec2-958b-c96a9127935b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2727-predictor-775569db54-5zrm6 namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: 73b3ca7a-0aeb-4dd4-bc64-f66f7c8abfe9 kind: Event lastTimestamp: "2026-09-09T20:13:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:25Z" name: success-200-isvc-d2727-predictor-775569db54-5zrm6.18d3c033b5d89a09 namespace: kserve-ci-e2e-test resourceVersion: "17597" uid: 91c35884-d6b1-46fa-ad44-a02d40abe1fa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-09T20:13:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2727-predictor-775569db54-5zrm6 namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: 73b3ca7a-0aeb-4dd4-bc64-f66f7c8abfe9 kind: Event lastTimestamp: "2026-09-09T20:22:41Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:41Z" name: success-200-isvc-d2727-predictor-775569db54-5zrm6.18d3c033c7f0e85b namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: 8bbb40b5-2dd2-4490-8692-88aad6bd9dbc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d2727-predictor-775569db54-5zrm6 namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: 73b3ca7a-0aeb-4dd4-bc64-f66f7c8abfe9 kind: Event lastTimestamp: "2026-09-09T20:22:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:39Z" name: success-200-isvc-d2727-predictor-775569db54-5zrm6.18d3c0b4b6002f67 namespace: kserve-ci-e2e-test resourceVersion: "21904" uid: 375eb797-fb6f-4cb6-8ada-09681a08cafa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2727-predictor-775569db54-5zrm6 namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: 73b3ca7a-0aeb-4dd4-bc64-f66f7c8abfe9 kind: Event lastTimestamp: "2026-09-09T20:22:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:39Z" name: success-200-isvc-d2727-predictor-775569db54-5zrm6.18d3c0b4b60157f1 namespace: kserve-ci-e2e-test resourceVersion: "21905" uid: e36dee9f-c34c-4f58-9c96-a9cf52165bec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d2727-predictor-775569db54-5zrm6 namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: 73b3ca7a-0aeb-4dd4-bc64-f66f7c8abfe9 kind: Event lastTimestamp: "2026-09-09T20:22:41Z" message: 'Readiness probe failed: Get "https://10.132.0.19:8643/healthz": dial tcp 10.132.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:22:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:41Z" name: success-200-isvc-d2727-predictor-775569db54-5zrm6.18d3c0b53c903fc3 namespace: kserve-ci-e2e-test resourceVersion: "22007" uid: 40f17113-6da8-45b0-b5d6-d8a8f7a12746 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d2727-predictor-775569db54 namespace: kserve-ci-e2e-test resourceVersion: "17505" uid: bd50a730-9f44-4c5e-904e-da473a7c2d54 kind: Event lastTimestamp: "2026-09-09T20:13:24Z" message: 'Created pod: success-200-isvc-d2727-predictor-775569db54-5zrm6' metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:24Z" name: success-200-isvc-d2727-predictor-775569db54.18d3c0338c4c168a namespace: kserve-ci-e2e-test resourceVersion: "17519" uid: 626a7b13-d78f-4fdd-820c-419a21595fa6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d2727-predictor namespace: kserve-ci-e2e-test resourceVersion: "17503" uid: de5cd644-a48c-4d5f-9ffe-dbe16a159f20 kind: Event lastTimestamp: "2026-09-09T20:13:24Z" message: Scaled up replica set success-200-isvc-d2727-predictor-775569db54 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:24Z" name: success-200-isvc-d2727-predictor.18d3c03388fa09f2 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: e977bf3e-6204-4b20-aacf-049fbdca5d88 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:13:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d2727-predictor namespace: kserve-ci-e2e-test resourceVersion: "17504" uid: 4d5e4a93-42c2-44df-83a1-9fb02ba38ae6 kind: Event lastTimestamp: "2026-09-09T20:13:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:54Z" name: success-200-isvc-d2727-predictor.18d3c037076f132c namespace: kserve-ci-e2e-test resourceVersion: "17872" uid: 21f7e617-acb6-4079-beb3-fd8c0c471f12 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:13:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d2727-predictor namespace: kserve-ci-e2e-test resourceVersion: "17504" uid: 4d5e4a93-42c2-44df-83a1-9fb02ba38ae6 kind: Event lastTimestamp: "2026-09-09T20:13:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:13:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:54Z" name: success-200-isvc-d2727-predictor.18d3c03707db8f4a namespace: kserve-ci-e2e-test resourceVersion: "17873" uid: 0fac2d26-5338-4e98-b086-364d08a1c334 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:14:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d2727-predictor namespace: kserve-ci-e2e-test resourceVersion: "17749" uid: 4d5e4a93-42c2-44df-83a1-9fb02ba38ae6 kind: Event lastTimestamp: "2026-09-09T20:14:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:14:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:39Z" name: success-200-isvc-d2727-predictor.18d3c03e054d4af1 namespace: kserve-ci-e2e-test resourceVersion: "18493" uid: 366499c8-7fe9-4e82-a09a-e2e13bbb3d55 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:14:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d2727-predictor namespace: kserve-ci-e2e-test resourceVersion: "17749" uid: 4d5e4a93-42c2-44df-83a1-9fb02ba38ae6 kind: Event lastTimestamp: "2026-09-09T20:14:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:14:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:39Z" name: success-200-isvc-d2727-predictor.18d3c03e05b209db namespace: kserve-ci-e2e-test resourceVersion: "18494" uid: d94b0407-662c-4e3e-a105-7f95d94a3f10 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:13:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d2727 namespace: kserve-ci-e2e-test resourceVersion: "17496" uid: 1041096d-eed7-456c-b649-faf29acd87e9 kind: Event lastTimestamp: "2026-09-09T20:14:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:14:11Z" name: success-200-isvc-d2727.18d3c03389057b1d namespace: kserve-ci-e2e-test resourceVersion: "18184" uid: d18509f4-6b02-43a1-9b85-5c49a259d6a3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d2727 namespace: kserve-ci-e2e-test resourceVersion: "18182" uid: 1041096d-eed7-456c-b649-faf29acd87e9 kind: Event lastTimestamp: "2026-09-09T20:14:11Z" message: InferenceService [success-200-isvc-d2727] is Ready metadata: creationTimestamp: "2026-09-09T20:14:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:14:11Z" name: success-200-isvc-d2727.18d3c03e84d26696 namespace: kserve-ci-e2e-test resourceVersion: "18183" uid: 994c1bfb-61d4-44c2-85d7-dcc75dcf98de reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:22:39.429257Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e029b-predictor-6b5b586b98-847mn namespace: kserve-ci-e2e-test resourceVersion: "21909" uid: a934850a-3086-4391-b93c-95b96ea4bdb9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e029b-predictor-6b5b586b98-847mn to ip-10-0-136-11.ec2.internal metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:22:39Z" name: success-200-isvc-e029b-predictor-6b5b586b98-847mn.18d3c0b4b927ede5 namespace: kserve-ci-e2e-test resourceVersion: "21919" uid: 97e7988a-cd0d-4d60-82be-2350ce86792e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e029b-predictor-6b5b586b98-847mn namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: a934850a-3086-4391-b93c-95b96ea4bdb9 kind: Event lastTimestamp: "2026-09-09T20:22:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e029b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:39Z" name: success-200-isvc-e029b-predictor-6b5b586b98-847mn.18d3c0b4c1ffb9fc namespace: kserve-ci-e2e-test resourceVersion: "21959" uid: e0875ff5-e388-4ca5-9cf0-ca902f5f83a8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e029b-predictor-6b5b586b98-847mn namespace: kserve-ci-e2e-test resourceVersion: "21936" uid: a934850a-3086-4391-b93c-95b96ea4bdb9 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:22:40Z" name: success-200-isvc-e029b-predictor-6b5b586b98-847mn.18d3c0b4fefe16d7 namespace: kserve-ci-e2e-test resourceVersion: "21984" uid: d7f7ec1d-fc30-49df-ba29-59d1f61487e7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e029b-predictor-6b5b586b98-847mn namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: a934850a-3086-4391-b93c-95b96ea4bdb9 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1907" already present on machine metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:40Z" name: success-200-isvc-e029b-predictor-6b5b586b98-847mn.18d3c0b50059b5fb namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 5176efd0-79f2-44d1-84db-ed8edcbac9ce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e029b-predictor-6b5b586b98-847mn namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: a934850a-3086-4391-b93c-95b96ea4bdb9 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:40Z" name: success-200-isvc-e029b-predictor-6b5b586b98-847mn.18d3c0b503907c08 namespace: kserve-ci-e2e-test resourceVersion: "21988" uid: a7636981-8c49-4524-b4ba-1720a2044289 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e029b-predictor-6b5b586b98-847mn namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: a934850a-3086-4391-b93c-95b96ea4bdb9 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:40Z" name: success-200-isvc-e029b-predictor-6b5b586b98-847mn.18d3c0b50401c6b0 namespace: kserve-ci-e2e-test resourceVersion: "21989" uid: 6588294c-077f-4fb5-8bb1-5098a6a9f20e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e029b-predictor-6b5b586b98-847mn namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: a934850a-3086-4391-b93c-95b96ea4bdb9 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:40Z" name: success-200-isvc-e029b-predictor-6b5b586b98-847mn.18d3c0b5040b782f namespace: kserve-ci-e2e-test resourceVersion: "21990" uid: 17035689-eb25-41e8-a55d-2e732d3f3be8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e029b-predictor-6b5b586b98-847mn namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: a934850a-3086-4391-b93c-95b96ea4bdb9 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:40Z" name: success-200-isvc-e029b-predictor-6b5b586b98-847mn.18d3c0b50708a115 namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 634a0fff-0004-42f1-b174-b39b09885390 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e029b-predictor-6b5b586b98-847mn namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: a934850a-3086-4391-b93c-95b96ea4bdb9 kind: Event lastTimestamp: "2026-09-09T20:22:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:40Z" name: success-200-isvc-e029b-predictor-6b5b586b98-847mn.18d3c0b5077b3c52 namespace: kserve-ci-e2e-test resourceVersion: "21992" uid: 94c3ba6d-67f0-45f5-b932-a2e20cd8b9c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:22:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e029b-predictor-6b5b586b98-847mn namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: a934850a-3086-4391-b93c-95b96ea4bdb9 kind: Event lastTimestamp: "2026-09-09T20:23:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:16Z" name: success-200-isvc-e029b-predictor-6b5b586b98-847mn.18d3c0b51083cdda namespace: kserve-ci-e2e-test resourceVersion: "22286" uid: 16ba36cc-b54d-4643-bd7c-e562f94cc781 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e029b-predictor-6b5b586b98-847mn namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: a934850a-3086-4391-b93c-95b96ea4bdb9 kind: Event lastTimestamp: "2026-09-09T20:23:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-09T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:49Z" name: success-200-isvc-e029b-predictor-6b5b586b98-847mn.18d3c0c510722ad5 namespace: kserve-ci-e2e-test resourceVersion: "22776" uid: 8cbcf1e5-5e48-4051-9ae8-9942175cf492 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e029b-predictor-6b5b586b98-847mn namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: a934850a-3086-4391-b93c-95b96ea4bdb9 kind: Event lastTimestamp: "2026-09-09T20:23:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-09T20:23:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:49Z" name: success-200-isvc-e029b-predictor-6b5b586b98-847mn.18d3c0c51072c366 namespace: kserve-ci-e2e-test resourceVersion: "22779" uid: a27fffe4-ee98-45a9-a8a1-9687cbcde698 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e029b-predictor-6b5b586b98-847mn namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: a934850a-3086-4391-b93c-95b96ea4bdb9 kind: Event lastTimestamp: "2026-09-09T20:23:51Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-09T20:23:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:51Z" name: success-200-isvc-e029b-predictor-6b5b586b98-847mn.18d3c0c59873b55f namespace: kserve-ci-e2e-test resourceVersion: "22892" uid: bbe90c02-b4e1-4e28-8acc-69725775cca8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-11.ec2.internal source: component: kubelet host: ip-10-0-136-11.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e029b-predictor-6b5b586b98 namespace: kserve-ci-e2e-test resourceVersion: "21891" uid: e3e89344-3e37-4c80-973d-875f9403ed99 kind: Event lastTimestamp: "2026-09-09T20:22:39Z" message: 'Created pod: success-200-isvc-e029b-predictor-6b5b586b98-847mn' metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:22:39Z" name: success-200-isvc-e029b-predictor-6b5b586b98.18d3c0b4b8d23e68 namespace: kserve-ci-e2e-test resourceVersion: "21914" uid: 041774a7-3e9b-4fa0-8e55-ef14800c54d7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e029b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21884" uid: 95adb9f9-d54b-4ec8-8794-576de85a993a kind: Event lastTimestamp: "2026-09-09T20:22:39Z" message: Scaled up replica set success-200-isvc-e029b-predictor-6b5b586b98 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:22:39Z" name: success-200-isvc-e029b-predictor.18d3c0b4b4df9e0c namespace: kserve-ci-e2e-test resourceVersion: "21896" uid: 7fc1626c-260f-405c-86c1-423213ce474b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:22:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e029b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: f35956ac-ec22-49ef-93b0-fb92c24190e0 kind: Event lastTimestamp: "2026-09-09T20:23:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:22:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:24Z" name: success-200-isvc-e029b-predictor.18d3c0b833750b45 namespace: kserve-ci-e2e-test resourceVersion: "22338" uid: cde29369-9241-4f46-b337-d8cb914aac9f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:22:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e029b-predictor namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: f35956ac-ec22-49ef-93b0-fb92c24190e0 kind: Event lastTimestamp: "2026-09-09T20:23:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:22:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:24Z" name: success-200-isvc-e029b-predictor.18d3c0b833fb83dd namespace: kserve-ci-e2e-test resourceVersion: "22339" uid: 4d2bb230-0a61-4893-82cc-aa8f045676e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:23:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e029b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22113" uid: f35956ac-ec22-49ef-93b0-fb92c24190e0 kind: Event lastTimestamp: "2026-09-09T20:23:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:39Z" name: success-200-isvc-e029b-predictor.18d3c0bbb31924af namespace: kserve-ci-e2e-test resourceVersion: "22624" uid: cc1ee33a-ad6f-4049-b346-1673ad870ffc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:23:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e029b-predictor namespace: kserve-ci-e2e-test resourceVersion: "22113" uid: f35956ac-ec22-49ef-93b0-fb92c24190e0 kind: Event lastTimestamp: "2026-09-09T20:23:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-09T20:23:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:23:39Z" name: success-200-isvc-e029b-predictor.18d3c0bbb3816a19 namespace: kserve-ci-e2e-test resourceVersion: "22630" uid: 28ec1b38-6a27-49b5-a5d5-5aaf104bc0ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:22:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e029b namespace: kserve-ci-e2e-test resourceVersion: "21876" uid: 5b9531b8-4a4f-431b-a596-1b4313de4d65 kind: Event lastTimestamp: "2026-09-09T20:23:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:23:27Z" name: success-200-isvc-e029b.18d3c0b4b4f3eca5 namespace: kserve-ci-e2e-test resourceVersion: "22512" uid: 624aed26-48ad-4022-83ce-7e73a1b4fda1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:23:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e029b namespace: kserve-ci-e2e-test resourceVersion: "22506" uid: 5b9531b8-4a4f-431b-a596-1b4313de4d65 kind: Event lastTimestamp: "2026-09-09T20:23:26Z" message: InferenceService [success-200-isvc-e029b] is Ready metadata: creationTimestamp: "2026-09-09T20:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:23:26Z" name: success-200-isvc-e029b.18d3c0bfcc38f5e8 namespace: kserve-ci-e2e-test resourceVersion: "22507" uid: 6e94a8f6-832f-4284-a580-401c11285902 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:13:10.264333Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3b763-7c7b5fc4cd-6jwx9 namespace: kserve-ci-e2e-test resourceVersion: "17321" uid: aaa306e6-d254-431c-ae55-157fbf136618 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-3b763-7c7b5fc4cd-6jwx9 to ip-10-0-128-182.ec2.internal metadata: creationTimestamp: "2026-09-09T20:13:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:13:10Z" name: switch-graph-3b763-7c7b5fc4cd-6jwx9.18d3c030344850b5 namespace: kserve-ci-e2e-test resourceVersion: "17327" uid: 55dd84e9-095b-41c2-a7f0-e0c2422fb53c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:10Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3b763-7c7b5fc4cd-6jwx9 namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: aaa306e6-d254-431c-ae55-157fbf136618 kind: Event lastTimestamp: "2026-09-09T20:13:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-3b763-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:10Z" name: switch-graph-3b763-7c7b5fc4cd-6jwx9.18d3c0303e5f73cf namespace: kserve-ci-e2e-test resourceVersion: "17339" uid: 726666ef-051c-4b75-a37f-190634f89c9d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3b763-7c7b5fc4cd-6jwx9 namespace: kserve-ci-e2e-test resourceVersion: "17330" uid: aaa306e6-d254-431c-ae55-157fbf136618 kind: Event lastTimestamp: "2026-09-09T20:13:11Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:13:11Z" name: switch-graph-3b763-7c7b5fc4cd-6jwx9.18d3c03079d8ea24 namespace: kserve-ci-e2e-test resourceVersion: "17348" uid: 2cf3af1e-d156-4163-89c0-141c3b938388 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3b763} kind: Pod name: switch-graph-3b763-7c7b5fc4cd-6jwx9 namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: aaa306e6-d254-431c-ae55-157fbf136618 kind: Event lastTimestamp: "2026-09-09T20:13:11Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:1c50c377478387906c73b97d7029d36701f6ae847fc2e48c877fa8c0c5df33ad" metadata: creationTimestamp: "2026-09-09T20:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:11Z" name: switch-graph-3b763-7c7b5fc4cd-6jwx9.18d3c0307b1e3c4b namespace: kserve-ci-e2e-test resourceVersion: "17351" uid: ad58e55c-36d2-4dcb-a816-6623b6389443 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3b763} kind: Pod name: switch-graph-3b763-7c7b5fc4cd-6jwx9 namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: aaa306e6-d254-431c-ae55-157fbf136618 kind: Event lastTimestamp: "2026-09-09T20:13:14Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:1c50c377478387906c73b97d7029d36701f6ae847fc2e48c877fa8c0c5df33ad" in 3.097s (3.097s including waiting). Image size: 214907971 bytes.' metadata: creationTimestamp: "2026-09-09T20:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:14Z" name: switch-graph-3b763-7c7b5fc4cd-6jwx9.18d3c03133c5c4b4 namespace: kserve-ci-e2e-test resourceVersion: "17373" uid: 5dd1b903-cf56-4bd5-9000-f1755aa8219f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3b763} kind: Pod name: switch-graph-3b763-7c7b5fc4cd-6jwx9 namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: aaa306e6-d254-431c-ae55-157fbf136618 kind: Event lastTimestamp: "2026-09-09T20:13:14Z" message: 'Created container: switch-graph-3b763' metadata: creationTimestamp: "2026-09-09T20:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:14Z" name: switch-graph-3b763-7c7b5fc4cd-6jwx9.18d3c0313723f8ba namespace: kserve-ci-e2e-test resourceVersion: "17375" uid: 541abd64-3494-4b1c-954f-b77ff7f50c6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3b763} kind: Pod name: switch-graph-3b763-7c7b5fc4cd-6jwx9 namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: aaa306e6-d254-431c-ae55-157fbf136618 kind: Event lastTimestamp: "2026-09-09T20:13:14Z" message: Started container switch-graph-3b763 metadata: creationTimestamp: "2026-09-09T20:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:14Z" name: switch-graph-3b763-7c7b5fc4cd-6jwx9.18d3c03137b26986 namespace: kserve-ci-e2e-test resourceVersion: "17376" uid: 94eedef3-3580-45a8-820d-f73058fadda3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3b763} kind: Pod name: switch-graph-3b763-7c7b5fc4cd-6jwx9 namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: aaa306e6-d254-431c-ae55-157fbf136618 kind: Event lastTimestamp: "2026-09-09T20:13:24Z" message: Stopping container switch-graph-3b763 metadata: creationTimestamp: "2026-09-09T20:13:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:24Z" name: switch-graph-3b763-7c7b5fc4cd-6jwx9.18d3c0337ece3a5d namespace: kserve-ci-e2e-test resourceVersion: "17487" uid: efb6970c-e96a-487d-a3e8-78cef4400c18 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:13:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3b763} kind: Pod name: switch-graph-3b763-7c7b5fc4cd-6jwx9 namespace: kserve-ci-e2e-test resourceVersion: "17324" uid: aaa306e6-d254-431c-ae55-157fbf136618 kind: Event lastTimestamp: "2026-09-09T20:13:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-09T20:13:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:13:51Z" name: switch-graph-3b763-7c7b5fc4cd-6jwx9.18d3c033de13c043 namespace: kserve-ci-e2e-test resourceVersion: "17841" uid: 1b29c8ab-e078-44d5-bcb1-5a78cd4db9f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-3b763-7c7b5fc4cd namespace: kserve-ci-e2e-test resourceVersion: "17317" uid: 0e68faf6-1ed4-4ec3-b338-54e1fc840e2b kind: Event lastTimestamp: "2026-09-09T20:13:10Z" message: 'Created pod: switch-graph-3b763-7c7b5fc4cd-6jwx9' metadata: creationTimestamp: "2026-09-09T20:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:10Z" name: switch-graph-3b763-7c7b5fc4cd.18d3c03033d8f80e namespace: kserve-ci-e2e-test resourceVersion: "17323" uid: 59b20aee-6e0f-46ca-8862-e234e87ad249 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-3b763 namespace: kserve-ci-e2e-test resourceVersion: "17315" uid: c79394df-5456-4518-ad68-9b5edb01a288 kind: Event lastTimestamp: "2026-09-09T20:13:10Z" message: Scaled up replica set switch-graph-3b763-7c7b5fc4cd from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:13:10Z" name: switch-graph-3b763.18d3c030329e3311 namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 06ccbbf5-7c57-44bd-bfcb-d5e3f6099f7e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:13:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3b763 namespace: kserve-ci-e2e-test resourceVersion: "17333" uid: edd3c126-0375-425c-aa2c-4157d6e4a77d kind: Event lastTimestamp: "2026-09-09T20:13:10Z" message: InferenceGraph [switch-graph-3b763] is Ready metadata: creationTimestamp: "2026-09-09T20:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:13:10Z" name: switch-graph-3b763.18d3c030385cd969 namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 40665ef3-7c51-4e63-8527-883845738adb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:35:10.554161Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-54686-7694d64fcc-shqw2 namespace: kserve-ci-e2e-test resourceVersion: "28908" uid: 0aa1113e-d6f2-45ae-8f69-4fefe0e5c09f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-54686-7694d64fcc-shqw2 to ip-10-0-128-182.ec2.internal metadata: creationTimestamp: "2026-09-09T20:35:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:35:10Z" name: switch-graph-54686-7694d64fcc-shqw2.18d3c1639bb0673b namespace: kserve-ci-e2e-test resourceVersion: "28913" uid: 118b3489-92f9-4b43-9bff-a64f8fba55b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:10Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-54686-7694d64fcc-shqw2 namespace: kserve-ci-e2e-test resourceVersion: "28911" uid: 0aa1113e-d6f2-45ae-8f69-4fefe0e5c09f kind: Event lastTimestamp: "2026-09-09T20:35:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-54686-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:35:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:35:10Z" name: switch-graph-54686-7694d64fcc-shqw2.18d3c163a8bc5dd7 namespace: kserve-ci-e2e-test resourceVersion: "28925" uid: 2292a39b-3989-48ec-a51f-a2f5ce9d11f6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:11Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-54686-7694d64fcc-shqw2 namespace: kserve-ci-e2e-test resourceVersion: "28917" uid: 0aa1113e-d6f2-45ae-8f69-4fefe0e5c09f kind: Event lastTimestamp: "2026-09-09T20:35:11Z" message: Add eth0 [10.133.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:35:11Z" name: switch-graph-54686-7694d64fcc-shqw2.18d3c163e1724989 namespace: kserve-ci-e2e-test resourceVersion: "28932" uid: 1673e4eb-e56f-4c44-b504-c610e8e729df reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-54686} kind: Pod name: switch-graph-54686-7694d64fcc-shqw2 namespace: kserve-ci-e2e-test resourceVersion: "28911" uid: 0aa1113e-d6f2-45ae-8f69-4fefe0e5c09f kind: Event lastTimestamp: "2026-09-09T20:35:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:1c50c377478387906c73b97d7029d36701f6ae847fc2e48c877fa8c0c5df33ad" already present on machine metadata: creationTimestamp: "2026-09-09T20:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:35:11Z" name: switch-graph-54686-7694d64fcc-shqw2.18d3c163e3058038 namespace: kserve-ci-e2e-test resourceVersion: "28934" uid: 86605671-7fc4-430c-8a97-ab4ca8cad2cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-54686} kind: Pod name: switch-graph-54686-7694d64fcc-shqw2 namespace: kserve-ci-e2e-test resourceVersion: "28911" uid: 0aa1113e-d6f2-45ae-8f69-4fefe0e5c09f kind: Event lastTimestamp: "2026-09-09T20:35:11Z" message: 'Created container: switch-graph-54686' metadata: creationTimestamp: "2026-09-09T20:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:35:11Z" name: switch-graph-54686-7694d64fcc-shqw2.18d3c163e5d4ecd5 namespace: kserve-ci-e2e-test resourceVersion: "28935" uid: a2921491-39d4-4124-b3ec-5de991c693ca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-54686} kind: Pod name: switch-graph-54686-7694d64fcc-shqw2 namespace: kserve-ci-e2e-test resourceVersion: "28911" uid: 0aa1113e-d6f2-45ae-8f69-4fefe0e5c09f kind: Event lastTimestamp: "2026-09-09T20:35:11Z" message: Started container switch-graph-54686 metadata: creationTimestamp: "2026-09-09T20:35:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:35:11Z" name: switch-graph-54686-7694d64fcc-shqw2.18d3c163e64aa422 namespace: kserve-ci-e2e-test resourceVersion: "28936" uid: 520ff4be-4be9-4b59-b0e6-882937a4a0fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:51:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-54686} kind: Pod name: switch-graph-54686-7694d64fcc-shqw2 namespace: kserve-ci-e2e-test resourceVersion: "28911" uid: 0aa1113e-d6f2-45ae-8f69-4fefe0e5c09f kind: Event lastTimestamp: "2026-09-09T20:51:29Z" message: Stopping container switch-graph-54686 metadata: creationTimestamp: "2026-09-09T20:51:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:51:29Z" name: switch-graph-54686-7694d64fcc-shqw2.18d3c24793536824 namespace: kserve-ci-e2e-test resourceVersion: "35808" uid: 119625ba-5b4f-4c42-bfb2-3c066a7d1739 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-09T20:51:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-54686} kind: Pod name: switch-graph-54686-7694d64fcc-shqw2 namespace: kserve-ci-e2e-test resourceVersion: "28911" uid: 0aa1113e-d6f2-45ae-8f69-4fefe0e5c09f kind: Event lastTimestamp: "2026-09-09T20:51:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-09T20:51:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:51:53Z" name: switch-graph-54686-7694d64fcc-shqw2.18d3c2485f96d6cd namespace: kserve-ci-e2e-test resourceVersion: "36051" uid: d5cabcca-1001-4ecb-b6f0-c22bb58a22a6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-54686-7694d64fcc namespace: kserve-ci-e2e-test resourceVersion: "28904" uid: 06e478e7-afe2-420b-9445-df3cf299e0bc kind: Event lastTimestamp: "2026-09-09T20:35:10Z" message: 'Created pod: switch-graph-54686-7694d64fcc-shqw2' metadata: creationTimestamp: "2026-09-09T20:35:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:10Z" name: switch-graph-54686-7694d64fcc.18d3c1639b4173a9 namespace: kserve-ci-e2e-test resourceVersion: "28910" uid: 57113e1a-cc59-44c5-9eba-276721c3b9a4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-54686 namespace: kserve-ci-e2e-test resourceVersion: "28902" uid: 036b11bd-bb30-4b7d-8641-cb8e89f919da kind: Event lastTimestamp: "2026-09-09T20:35:10Z" message: Scaled up replica set switch-graph-54686-7694d64fcc from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:35:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:10Z" name: switch-graph-54686.18d3c1639a42af13 namespace: kserve-ci-e2e-test resourceVersion: "28907" uid: ff697b04-1826-47aa-8303-4700fa825d81 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:35:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-54686 namespace: kserve-ci-e2e-test resourceVersion: "28919" uid: 7580cc8a-3696-4a25-a255-727cea10daab kind: Event lastTimestamp: "2026-09-09T20:35:10Z" message: InferenceGraph [switch-graph-54686] is Ready metadata: creationTimestamp: "2026-09-09T20:35:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:35:10Z" name: switch-graph-54686.18d3c1639e84b59f namespace: kserve-ci-e2e-test resourceVersion: "28921" uid: 2579daa9-57e1-47ea-89b2-351fde0158e2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:35:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-54686 namespace: kserve-ci-e2e-test resourceVersion: "28903" uid: e43d16c1-4532-42c3-8d63-c4a0aca7b5b5 kind: Event lastTimestamp: "2026-09-09T20:35:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:40Z" name: switch-graph-54686.18d3c16719495977 namespace: kserve-ci-e2e-test resourceVersion: "29216" uid: 60134307-c240-4232-8222-2ad232e0b6b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-09T20:35:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-54686 namespace: kserve-ci-e2e-test resourceVersion: "28903" uid: e43d16c1-4532-42c3-8d63-c4a0aca7b5b5 kind: Event lastTimestamp: "2026-09-09T20:35:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:35:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:35:40Z" name: switch-graph-54686.18d3c16719b3e794 namespace: kserve-ci-e2e-test resourceVersion: "29217" uid: b1fefbb9-905b-4675-b305-9f483dae9c0b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-09T20:14:24.619642Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d2727-58876698f6-whzzh namespace: kserve-ci-e2e-test resourceVersion: "18343" uid: 2621a172-058f-462e-b09b-8d3979246515 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d2727-58876698f6-whzzh to ip-10-0-128-182.ec2.internal metadata: creationTimestamp: "2026-09-09T20:14:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-09T20:14:24Z" name: switch-graph-d2727-58876698f6-whzzh.18d3c04184345be8 namespace: kserve-ci-e2e-test resourceVersion: "18348" uid: 29d9b7fd-028a-49c9-9704-712d0825ec2e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-b8f5446f6-xzjjx source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:24Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d2727-58876698f6-whzzh namespace: kserve-ci-e2e-test resourceVersion: "18345" uid: 2621a172-058f-462e-b09b-8d3979246515 kind: Event lastTimestamp: "2026-09-09T20:14:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-d2727-serving-cert" not found' metadata: creationTimestamp: "2026-09-09T20:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:24Z" name: switch-graph-d2727-58876698f6-whzzh.18d3c0419375729d namespace: kserve-ci-e2e-test resourceVersion: "18364" uid: 40d3fc95-ac6c-4637-aa27-24c94b677cd9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:25Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d2727-58876698f6-whzzh namespace: kserve-ci-e2e-test resourceVersion: "18355" uid: 2621a172-058f-462e-b09b-8d3979246515 kind: Event lastTimestamp: "2026-09-09T20:14:25Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-09T20:14:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-09T20:14:25Z" name: switch-graph-d2727-58876698f6-whzzh.18d3c041ca131bb0 namespace: kserve-ci-e2e-test resourceVersion: "18378" uid: bb808286-1a0d-4f66-894d-488bac11823e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d2727} kind: Pod name: switch-graph-d2727-58876698f6-whzzh namespace: kserve-ci-e2e-test resourceVersion: "18345" uid: 2621a172-058f-462e-b09b-8d3979246515 kind: Event lastTimestamp: "2026-09-09T20:14:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:1c50c377478387906c73b97d7029d36701f6ae847fc2e48c877fa8c0c5df33ad" already present on machine metadata: creationTimestamp: "2026-09-09T20:14:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:25Z" name: switch-graph-d2727-58876698f6-whzzh.18d3c041cb597e67 namespace: kserve-ci-e2e-test resourceVersion: "18380" uid: a7a8a13b-33ed-4ce8-b2db-467ce78c6724 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d2727} kind: Pod name: switch-graph-d2727-58876698f6-whzzh namespace: kserve-ci-e2e-test resourceVersion: "18345" uid: 2621a172-058f-462e-b09b-8d3979246515 kind: Event lastTimestamp: "2026-09-09T20:14:25Z" message: 'Created container: switch-graph-d2727' metadata: creationTimestamp: "2026-09-09T20:14:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:25Z" name: switch-graph-d2727-58876698f6-whzzh.18d3c041ce923ff9 namespace: kserve-ci-e2e-test resourceVersion: "18382" uid: 0ab39b3c-0805-438b-99f7-d3f66b14f29b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d2727} kind: Pod name: switch-graph-d2727-58876698f6-whzzh namespace: kserve-ci-e2e-test resourceVersion: "18345" uid: 2621a172-058f-462e-b09b-8d3979246515 kind: Event lastTimestamp: "2026-09-09T20:14:25Z" message: Started container switch-graph-d2727 metadata: creationTimestamp: "2026-09-09T20:14:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:14:25Z" name: switch-graph-d2727-58876698f6-whzzh.18d3c041cf0ab461 namespace: kserve-ci-e2e-test resourceVersion: "18383" uid: 872f0f47-baec-4f11-8d46-8c817a02be7d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:22:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d2727} kind: Pod name: switch-graph-d2727-58876698f6-whzzh namespace: kserve-ci-e2e-test resourceVersion: "18345" uid: 2621a172-058f-462e-b09b-8d3979246515 kind: Event lastTimestamp: "2026-09-09T20:22:39Z" message: Stopping container switch-graph-d2727 metadata: creationTimestamp: "2026-09-09T20:22:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:22:39Z" name: switch-graph-d2727-58876698f6-whzzh.18d3c0b4acb6b0a8 namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: f334f3f9-ad6d-4d22-9c19-e7b547a0c429 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-09T20:22:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d2727} kind: Pod name: switch-graph-d2727-58876698f6-whzzh namespace: kserve-ci-e2e-test resourceVersion: "18345" uid: 2621a172-058f-462e-b09b-8d3979246515 kind: Event lastTimestamp: "2026-09-09T20:23:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-09T20:22:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-09T20:23:07Z" name: switch-graph-d2727-58876698f6-whzzh.18d3c0b566b66a5e namespace: kserve-ci-e2e-test resourceVersion: "22214" uid: e97b5f08-7480-4ce5-862f-6e9a67fc7be8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-182.ec2.internal source: component: kubelet host: ip-10-0-128-182.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d2727-58876698f6 namespace: kserve-ci-e2e-test resourceVersion: "18340" uid: 80122571-db7a-4703-8edc-c405cc6faf51 kind: Event lastTimestamp: "2026-09-09T20:14:24Z" message: 'Created pod: switch-graph-d2727-58876698f6-whzzh' metadata: creationTimestamp: "2026-09-09T20:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:24Z" name: switch-graph-d2727-58876698f6.18d3c04183cf95cc namespace: kserve-ci-e2e-test resourceVersion: "18344" uid: f79b2302-05fb-47e0-86c4-602b6b02e1d8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d2727 namespace: kserve-ci-e2e-test resourceVersion: "18338" uid: 6f472d4f-9577-45bc-b7d2-320c5fc83193 kind: Event lastTimestamp: "2026-09-09T20:14:24Z" message: Scaled up replica set switch-graph-d2727-58876698f6 from 0 to 1 metadata: creationTimestamp: "2026-09-09T20:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:14:24Z" name: switch-graph-d2727.18d3c0418316f797 namespace: kserve-ci-e2e-test resourceVersion: "18341" uid: 8c4e1c56-fc1e-44ce-86b3-b2d35224bac5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-09T20:14:24Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d2727 namespace: kserve-ci-e2e-test resourceVersion: "18357" uid: 721a37e8-be9b-4f59-8027-8b9895c8cf92 kind: Event lastTimestamp: "2026-09-09T20:14:24Z" message: InferenceGraph [switch-graph-d2727] is Ready metadata: creationTimestamp: "2026-09-09T20:14:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-09T20:14:24Z" name: switch-graph-d2727.18d3c0418872a8a7 namespace: kserve-ci-e2e-test resourceVersion: "18359" uid: 821eaada-06d7-4ce6-898d-b5095066f660 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:14:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d2727 namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: 960d2efd-4f98-4fdc-8de9-99f85fe078c4 kind: Event lastTimestamp: "2026-09-09T20:15:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:14:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:15:09Z" name: switch-graph-d2727.18d3c04501faf063 namespace: kserve-ci-e2e-test resourceVersion: "18733" uid: 22d910d6-f94c-49ce-af0c-aede4fd0d5fc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-09T20:14:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d2727 namespace: kserve-ci-e2e-test resourceVersion: "18339" uid: 960d2efd-4f98-4fdc-8de9-99f85fe078c4 kind: Event lastTimestamp: "2026-09-09T20:15:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-09T20:14:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-09T20:15:09Z" name: switch-graph-d2727.18d3c045025c7495 namespace: kserve-ci-e2e-test resourceVersion: "18734" uid: 5a148f39-f2dc-4af6-ba05-c9fa93b315e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "36076"